[feature][m]: add full publisher report functionality (#11)

* [feature][m]: add full publisher report functionality

* [fix][m]: review fixes

* [fix][xs]: add comment for clarity on report CSV file naming

ckanext/opendatani/controller.py

@@ -1,18 +1,22 @@
+import logging
+import datetime as dt
+import requests
+import os
+import csv
+import cStringIO as StringIO
+
 import ckan.lib.base as base
 from ckan import model
 import ckan.lib.helpers as h
 import ckan.lib.mailer as mailer
 from ckan.common import c, request, _
 from ckan.common import config
-
 import ckan.plugins.toolkit as toolkit
 from ckan.controllers.user import UserController as CoreUserController
 from ckan.controllers.package import PackageController as CorePackageController
-
-import logging
 import ckan.logic as logic
-import datetime as dt
-import requests
+from ckanext.opendatani import helpers
+
 
 log = logging.getLogger(__name__)
 
@@ -249,3 +253,62 @@ def resource_read(self, id, resource_id):
                     h.flash_error('Sorry, this file is too large to be able to display in the browser, please download the data resource to examine it further.')
         template = self._resource_template(dataset_type)
         return render(template, extra_vars=vars)
+
+
+class CustomReportController(CorePackageController):
+    def prepare_report(self, org):
+        """
+        Creates a CSV publisher report and returns it as a string
+        :param org: organization
+        :type org: string
+        :return: a CSV string
+        :rtype: string
+        """
+
+        try:
+            data_dict = {'org_name': org}
+
+            if org == '@complete':
+                data_dict = {}
+                # Use 'complete' in the file name for full reports
+                org = org[1:]
+            else:
+                # Use 'org-' in the file name for per org reports
+                org = 'org-' + org
+
+            resource = toolkit.get_action(
+                'report_resources_by_organization')({}, data_dict)
+
+            # We need this in case no datasets exist for the given organization
+            # or the organization doesn't exist
+            if not resource:
+                toolkit.abort(404, _('Either the organization does not exist, \
+                                     or it has no datasets.'))
+
+            csvout = StringIO.StringIO()
+            csvwriter = csv.writer(
+                csvout,
+                dialect='excel',
+                quoting=csv.QUOTE_NONNUMERIC
+            )
+
+            fields = resource[0].keys()
+            csvwriter.writerow(fields)
+
+            for data in resource:
+                csvwriter.writerow(data.values())
+
+            csvout.seek(0)
+            filename = 'publisher-report-{0}-{1}.csv'.format(org,
+                                                             dt.date.today())
+            toolkit.response.headers['Content-Type'] = 'application/csv'
+            toolkit.response.headers['Content-Disposition'] = \
+                'attachment; filename={0}'.format(filename)
+
+            return csvout.read()
+
+        except Exception as ex:
+            error = 'Preparing the CSV report failed. Error: {0}'.format(ex)
+            log.error(error)
+            h.flash_error(error)
+            raise

ckanext/opendatani/helpers.py

@@ -1,19 +1,15 @@
 import re
 from six import string_types, text_type
+import logging
 
 import ckan.model as model
 from ckan.lib import activity_streams
 import ckan.logic as logic
 import ckan.lib.helpers as h
-
-import logging
 from ckan.plugins import toolkit
-from ckan.common import config
-import csv
-import json
-import os
 import ckan.authz as authz
 
+
 log = logging.getLogger(__name__)
 
 
@@ -124,8 +120,8 @@ def _get_action(action, context_dict, data_dict):
 
 def is_admin(user, org):
     """
-    Returns True if user is site admin or admin of the organization,
-    and the given organization exists.
+    Returns True if user is site admin or admin of the
+    organization and the given organization exists
     :param user: user name
     :type user: string
     :param org: organization name
@@ -140,70 +136,23 @@ def is_admin(user, org):
 
     return any(
         [(i.get('capacity') == 'admin')
-         and i.get('name') == org for i in user_orgs])\
+         and i.get('name') == org for i in user_orgs]) \
         or authz.is_sysadmin(user)
 
 
 def verify_datasets_exist(org):
     """
-    Returns True if the number of datasets (including private) for a given
-    organization is greater than 0.
+    Returns True if the number of datasets (including private)
+    for a given organization is greater than 0.
     :param org: organization name
     :type org: string
     :returns: dataset count
     :rtype: integer
     """
 
-    return toolkit.get_action('package_search')({}, {
-        'q': 'organization:{0}'.format(org),
-        'include_private': True}).get('count') > 0
-
-
-def prepare_reports(org):
-    """
-    Creates a CSV and JSON publisher report, and stores them under CKAN's
-    storage path in /storage/publisher-reports/.
-    :param org: organization
-    :type org: string
-    :return: a list containing the file_names of the created archives
-    :rtype: list
-    """
-
-    resource = toolkit.get_action(
-        'report_resources_by_organization')({}, {'org_name': org})
-    file_names = []
-    storage_path = config.get('ckan.storage_path')
-    file_path = storage_path + '/storage/publisher-reports/'
-
-    if not os.path.exists(file_path):
-        os.makedirs(file_path)
-
-    for file_type in ['.csv', '.json']:
-        try:
-            file_name = 'publisher-report-' + org + file_type
-
-            if file_type == '.csv':
-                with open(file_path + file_name, 'w') as csvfile:
-                    fields = resource[0].keys()
-                    writer = csv.DictWriter(csvfile, fieldnames=fields,
-                                            quoting=csv.QUOTE_MINIMAL)
-                    writer.writeheader()
-
-                    for data in resource:
-                        writer.writerow(data)
-
-                file_names.append(file_name)
-
-            if file_type == '.json':
-                with open(file_path + file_name, 'w') as jsonfile:
-                    jsonfile.writelines(json.dumps(resource))
-
-                file_names.append(file_name)
+    data_dict = {'include_private': True}
 
-        except Exception as ex:
-            log.error(
-                'An error occured while preparing the {0} archive. Error: {1}'
-                .format(file_type, ex))
-            raise
+    if org:
+        data_dict['q'] = 'organization:{0}'.format(org)
 
-    return file_names
+    return toolkit.get_action('package_search')({}, data_dict).get('count') > 0

ckanext/opendatani/plugin.py

@@ -1,6 +1,7 @@
 import datetime
 from pylons import config
 import routes.mapper
+import logging
 
 import ckan.plugins as plugins
 import ckan.plugins.toolkit as toolkit
@@ -12,9 +13,8 @@
 import datetime as dt
 from ckanext.opendatani.controller import CustomUserController
 from ckanext.opendatani import helpers
-
 from ckan.common import OrderedDict
-import logging
+
 
 log = logging.getLogger(__name__)
 
@@ -66,8 +66,7 @@ def get_helpers(self):
             'package_list': package_list,
             'ni_activity_list_to_text': helpers.activity_list_to_text,
             'verify_datasets_exist': helpers.verify_datasets_exist,
-            'is_admin': helpers.is_admin,
-            'prepare_reports': helpers.prepare_reports,
+            'is_admin': helpers.is_admin
         }
 
     # IRoutes
@@ -100,6 +99,11 @@ def before_map(self, map):
             m.connect('/dataset/{id}/resource/{resource_id}',
                       action='resource_read')
 
+        controller = 'ckanext.opendatani.controller:CustomReportController'
+        with routes.mapper.SubMapper(map, controller=controller) as m:
+            m.connect('/publisher-reports/publisher-report-{org}.csv',
+                      action='prepare_report')
+
         return map
 
     def after_map(self, map):
@@ -182,7 +186,10 @@ def report_resources_by_organization(context, data_dict):
                       report or the organization does not exist.'))
 
     data_dict['include_private'] = True
-    data_dict['q'] = 'organization:{0}'.format(org)
+
+    if org:
+        data_dict['q'] = 'organization:{0}'.format(org)
+
     results = toolkit.get_action('package_search')({}, data_dict)
 
     for item in results['results']:

ckanext/opendatani/templates/package/search.html

@@ -0,0 +1,13 @@
+{% ckan_extends %}
+
+{% block secondary_content %}
+  {% block report %}
+    {% if request.path[-7:] == 'dataset' and h.is_admin(c.user, org) and h.verify_datasets_exist(org) %}
+      <dl>
+        <h6 class="heading">{{ _('Full Publisher Report') }}</h6>
+        <a class="btn btn-success" href="{{ '/publisher-reports/[email protected]' }}">CSV</a>
+      </dl>
+    {% endif %}
+  {% endblock %}
+{{ super() }}
+{% endblock %}

ckanext/opendatani/templates/snippets/organization.html

@@ -5,11 +5,9 @@
   {% block report %}
   {% set org = request.path.replace('/organization/', '') %}
     {% if 'organization' in request.path and h.is_admin(c.user, org) and h.verify_datasets_exist(org) %}
-      {% set csv, json = h.prepare_reports(org) %}
       <dl>
-        <h1 class="heading">{{ _('Publisher Report') }}</h1>
-        <a class="btn btn-primary" href="{{ '/publisher-reports/' + csv }}" download="{{ csv }}">CSV</a>
-        <a class="btn btn-primary" href="{{ '/publisher-reports/' + json }}" download="{{ json }}">JSON</a>
+        <h6 class="heading">{{ _('Publisher Report') }}</h6>
+        <a class="btn btn-success" href="{{ '/publisher-reports/publisher-report-' + org + '.csv' }}">CSV</a>
       </dl>
     {% endif %}
   {% endblock %}