| from types import * |
| import json |
| import logging |
| import urllib.request, urllib.parse, urllib.error |
| |
| import requestresponder |
| from edge.dateutility import DateUtility |
| from edge.httputility import HttpUtility |
| from edge.opensearch.responsewriter import ResponseWriter |
| import re |
| |
| class DatasetWriter(ResponseWriter): |
| def __init__(self, configFilePath): |
| super(DatasetWriter, self).__init__(configFilePath) |
| self.datasets = [] |
| |
| def get(self, requestHandler): |
| super(DatasetWriter, self).get(requestHandler) |
| #searchParameters = {} |
| #logging.debug('uri: '+str(requestHandler.request.headers)) |
| |
| startIndex = 0 |
| try: |
| startIndex = requestHandler.get_argument('startIndex') |
| except: |
| pass |
| self.searchParameters['startIndex'] = startIndex |
| |
| entriesPerPage = self._configuration.getint('solr', 'entriesPerPage') |
| try: |
| entriesPerPage = requestHandler.get_argument('itemsPerPage') |
| #cap entries per age at 400 |
| if (int(entriesPerPage) > 400): |
| entriesPerPage = 400 |
| except: |
| pass |
| self.searchParameters['itemsPerPage'] = entriesPerPage |
| |
| #pretty = True |
| try: |
| if requestHandler.get_argument('pretty').lower() == 'false': |
| self.pretty = False |
| self.searchParameters['pretty'] = 'false' |
| except: |
| pass |
| |
| try: |
| if requestHandler.get_argument('full').lower() == 'true': |
| self.searchParameters['full'] = 'true' |
| except: |
| pass |
| |
| try: |
| self.searchParameters['format'] = requestHandler.get_argument('format') |
| except: |
| pass |
| |
| parameters = ['startTime', 'endTime', 'keyword', 'identifier', 'shortName', 'instrument', 'platform', 'fileFormat', 'status', 'processLevel', 'sortBy', 'bbox', 'allowNone'] |
| #variables = {} |
| for parameter in parameters: |
| try: |
| value = requestHandler.get_argument(parameter) |
| self.variables[parameter] = value |
| self.searchParameters[parameter] = value |
| except: |
| pass |
| |
| if 'keyword' in self.variables: |
| self.variables['keyword'] = self.variables['keyword'].replace('*', '') |
| self.variables['keyword'] = self.variables['keyword'].lower() |
| """ |
| else: |
| variables['keyword'] = '""' |
| """ |
| #If generating OpenSearch response, need to make additional call to solr |
| #to determine which datasets have granules |
| try: |
| if 'search' in requestHandler.request.path: |
| callback = self._getHasGranuleResponseCallback(startIndex, entriesPerPage) |
| self._getHasGranuleResponse(callback) |
| else: |
| self._getResponse(startIndex, entriesPerPage, self.variables) |
| except: |
| logging.exception('Failed to get solr response.') |
| """ |
| searchText = '' |
| if 'keyword' in variables: |
| searchText = variables['keyword'] |
| openSearchResponse = self._generateOpenSearchResponse( |
| solrResponse, |
| searchText, |
| self._configuration.get('service', 'url') + requestHandler.request.path, |
| searchParameters, |
| pretty |
| ) |
| |
| requestHandler.set_header("Content-Type", "application/xml") |
| #requestHandler.set_header("Content-Type", "application/rss+xml") |
| #requestHandler.write(solrResponse) |
| requestHandler.write(openSearchResponse) |
| """ |
| |
| def _getResponse(self, startIndex, entriesPerPage, variables): |
| query = self._constructSolrQuery(startIndex, entriesPerPage, variables) |
| url = self._configuration.get('solr', 'datasetUrl') |
| |
| httpUtility = HttpUtility() |
| httpUtility.getResponse(url+'/_search/?'+query, self._onSolrResponse) |
| |
| def _constructSolrQuery(self, startIndex, entriesPerPage, variables): |
| queries = [] |
| sort = None |
| filterQuery = None |
| for key, value in variables.items(): |
| #query = '' |
| if key == 'startTime': |
| startTime = DateUtility.convertISOToUTCTimestamp(value) |
| if startTime is not None: |
| query = 'stop_time:' |
| query += '['+str(startTime)+'%20TO%20*]' |
| queries.append(query) |
| elif key == 'endTime': |
| stopTime = DateUtility.convertISOToUTCTimestamp(value) |
| if stopTime is not None: |
| query = 'start_time:' |
| query += '[*%20TO%20'+str(stopTime)+']' |
| queries.append(query) |
| elif key == 'keyword': |
| newValue = urllib.parse.quote(value) |
| |
| query = newValue |
| queries.append(query) |
| elif key == 'identifier': |
| query = 'identifier:'+self._urlEncodeSolrQueryValue(value) |
| queries.append(query) |
| elif key == 'shortName': |
| query = 'Dataset-ShortName-Full:'+self._urlEncodeSolrQueryValue(value) |
| queries.append(query) |
| elif key == 'platform': |
| query = 'platform:'+self._urlEncodeSolrQueryValue(value) |
| queries.append(query) |
| elif key == 'instrument': |
| query = 'instrument:'+self._urlEncodeSolrQueryValue(value) |
| queries.append(query) |
| elif key == 'fileFormat': |
| query = 'DatasetPolicy-DataFormat-LowerCased:'+self._urlEncodeSolrQueryValue(value) |
| queries.append(query) |
| elif key == 'status': |
| query = 'DatasetPolicy-AccessType-LowerCased:'+self._urlEncodeSolrQueryValue(value) |
| queries.append(query) |
| elif key == 'processLevel': |
| query = 'Dataset-ProcessingLevel-LowerCased:'+value |
| queries.append(query) |
| elif key == 'sortBy': |
| sortByMapping = {'timeDesc': 'start_time:desc', 'timeAsc': 'start_time:asc'} |
| if value in list(sortByMapping.keys()): |
| sort = sortByMapping[value] |
| elif key == 'bbox': |
| filterQuery = self._constructBoundingBoxQuery(value) |
| |
| #if query != '': |
| # queries.append('%2B'+query) |
| |
| if len(queries) == 0: |
| queries.append('*') |
| |
| query = 'q='+'+AND+'.join(queries)+'&from='+str(startIndex)+'&size='+str(entriesPerPage) |
| if sort is not None: |
| query += '&sort=' + sort |
| if filterQuery is not None: |
| query += '&' + filterQuery |
| logging.debug('solr query: '+query) |
| |
| return query |
| |
| def _getHasGranuleResponse(self, callback): |
| url = self._configuration.get('solr', 'granuleUrl') |
| |
| httpUtility = HttpUtility() |
| return httpUtility.getResponse(url+'/_search', callback, '{"query" : {"match_all" : {}}, "size" : 0, "facets" : { "identifier" : { "terms" : {"field" : "identifier"}}}}') |
| |
| def _getHasGranuleResponseCallback(self, startIndex, entriesPerPage): |
| def onSolrHasGranuleResponse(response): |
| try: |
| solrJson = json.loads(response.body) |
| logging.debug("Got response for dataset facet") |
| facets = solrJson['facets']['identifier']['terms'] |
| self.datasets = [facet['term'] for facet in facets] |
| self._getResponse(startIndex, entriesPerPage, self.variables) |
| except: |
| logging.exception('Failed to get solr response.') |
| return onSolrHasGranuleResponse |