From 9938157217fe4b089b63ab18b25c687bf4ca2c01 Mon Sep 17 00:00:00 2001 From: Joon Lee Date: Mon, 9 Feb 2026 17:49:00 +0000 Subject: [PATCH 1/9] Error handling for Opensearch response and changed default sort --- Controllers/dataset.controllers.js | 15 ++++++++++++--- Services/dataset.service.js | 11 ++++++++++- Utils/datasetFields.js | 2 ++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Controllers/dataset.controllers.js b/Controllers/dataset.controllers.js index 3231bda..4b6fa91 100644 --- a/Controllers/dataset.controllers.js +++ b/Controllers/dataset.controllers.js @@ -3,7 +3,7 @@ const cache = require('../Components/cache'); const config = require('../Config'); const path = require('path'); const { Parser } = require('json2csv'); -const { datasetFields } = require('../Utils/datasetFields'); +const { DATASET_DEFAULT_SORT_FIELD, datasetFields } = require('../Utils/datasetFields'); const datasetService = require('../Services/dataset.service'); const search = async (req, res) => { @@ -13,7 +13,7 @@ const search = async (req, res) => { const options = {}; const pageInfo = body.pageInfo ?? {page: 1, pageSize: 10}; const searchText = body.search_text?.trim() ?? ''; - const sort = body.sort ?? {k: 'dbGaP_phs', v: 'asc'}; + const sort = body.sort ?? {k: DATASET_DEFAULT_SORT_FIELD, v: 'asc'}; if (pageInfo.page !== parseInt(pageInfo.page, 10) || pageInfo.page <= 0) { pageInfo.page = 1; @@ -37,7 +37,7 @@ const search = async (req, res) => { // sort.name = "Resource"; // sort.k = "data_resource_id"; // } - if (!(sort.v && ['asc', 'desc'].includes(sort.v))) { + if (sort?.v && !['asc', 'desc'].includes(sort?.v)) { sort.v = 'asc'; } @@ -47,6 +47,15 @@ const search = async (req, res) => { data.pageInfo = options.pageInfo; const searchResult = await datasetService.search(searchText, filters, options); + if (searchResult.error) { + res.json({ + status:"error", + aggs: 'all', + data: {}, + error: searchResult.error, + }); + return; + } if (searchResult.total !== 0 && (options.pageInfo.page - 1) * options.pageInfo.pageSize >= searchResult.total) { let lastPage = Math.ceil(searchResult.total / options.pageInfo.pageSize); diff --git a/Services/dataset.service.js b/Services/dataset.service.js index fee79c5..2117264 100644 --- a/Services/dataset.service.js +++ b/Services/dataset.service.js @@ -1,6 +1,7 @@ const config = require('../Config'); const elasticsearch = require('../Components/elasticsearch'); const cache = require('../Components/cache'); +const logger = require('../Components/logger'); const mysql = require('../Components/mysql'); const queryGenerator = require('./queryGenerator'); const cacheKeyGenerator = require('./cacheKeyGenerator'); @@ -16,6 +17,7 @@ const search = async (searchText, filters, options) => { let query = null; let result = {}; let searchableText = null; + let searchResults; // Check searchText type if (searchText && typeof searchText !== 'string') { @@ -60,7 +62,14 @@ const search = async (searchText, filters, options) => { result.aggs = 'all'; } - let searchResults = await elasticsearch.searchWithAggregations(config.indexDS, query); + try { + searchResults = await elasticsearch.searchWithAggregations(config.indexDS, query); + } catch (error) { + logger.error(`Error searching datasets: ${error}`); + return { + error: error?.body?.error?.root_cause ? JSON.stringify(error.body.error.root_cause).replace(/\\n/g, '') : error.message, + }; + } let datasets = searchResults.hits.hits.map((ds) => { if (ds.inner_hits) { const terms = Object.keys(ds.inner_hits); diff --git a/Utils/datasetFields.js b/Utils/datasetFields.js index 4596224..5e7e799 100644 --- a/Utils/datasetFields.js +++ b/Utils/datasetFields.js @@ -1,3 +1,4 @@ +const DATASET_DEFAULT_SORT_FIELD = 'dataset_title_sort'; // Maps Dataset natural field names to property names const DATASET_SEARCH_FIELDS = [ // 'dataset_uuid', @@ -93,6 +94,7 @@ const datasetFields = { }; module.exports = { + DATASET_DEFAULT_SORT_FIELD: 'dataset_title_sort', DATASET_SEARCH_FIELDS, DATASET_HIGHLIGHT_FIELDS, DATASET_RETURN_FIELDS, From ef4f249f4a32c3dc3308edb838e37d63f62ddce3 Mon Sep 17 00:00:00 2001 From: Joon Lee Date: Mon, 9 Feb 2026 17:54:10 +0000 Subject: [PATCH 2/9] Added input validation for controller's search function --- Controllers/dataset.controllers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Controllers/dataset.controllers.js b/Controllers/dataset.controllers.js index 4b6fa91..22783ef 100644 --- a/Controllers/dataset.controllers.js +++ b/Controllers/dataset.controllers.js @@ -9,10 +9,10 @@ const datasetService = require('../Services/dataset.service'); const search = async (req, res) => { const body = req.body; const data = {}; - const filters = body.filters ?? {}; + const filters = body.filters && typeof body.filters === 'object' && !Array.isArray(body.filters) ? body.filters : {}; const options = {}; const pageInfo = body.pageInfo ?? {page: 1, pageSize: 10}; - const searchText = body.search_text?.trim() ?? ''; + const searchText = body.search_text && typeof body.search_text === 'string' ? body.search_text.trim() : ''; const sort = body.sort ?? {k: DATASET_DEFAULT_SORT_FIELD, v: 'asc'}; if (pageInfo.page !== parseInt(pageInfo.page, 10) || pageInfo.page <= 0) { From 86ffb3342f3e16853d67cfea0835e194bc823016 Mon Sep 17 00:00:00 2001 From: Joon Lee Date: Mon, 9 Feb 2026 18:12:19 +0000 Subject: [PATCH 3/9] Added test for handling Opensearch error response for /search --- Services/dataset.service.js | 1 + Services/dataset.service.test.fixtures.js | 32 +++++++++++++++++++++++ Services/dataset.service.test.js | 11 ++++++++ 3 files changed, 44 insertions(+) diff --git a/Services/dataset.service.js b/Services/dataset.service.js index 2117264..f1508ab 100644 --- a/Services/dataset.service.js +++ b/Services/dataset.service.js @@ -70,6 +70,7 @@ const search = async (searchText, filters, options) => { error: error?.body?.error?.root_cause ? JSON.stringify(error.body.error.root_cause).replace(/\\n/g, '') : error.message, }; } + let datasets = searchResults.hits.hits.map((ds) => { if (ds.inner_hits) { const terms = Object.keys(ds.inner_hits); diff --git a/Services/dataset.service.test.fixtures.js b/Services/dataset.service.test.fixtures.js index c5407ce..a20c2df 100644 --- a/Services/dataset.service.test.fixtures.js +++ b/Services/dataset.service.test.fixtures.js @@ -180,3 +180,35 @@ export const normalOpensearchResults = { }, aggs: undefined, }; + +// Example of Opensearch response with error +export const errorOpensearchResults = { + "error": { + "root_cause": [ + { + "type": "query_shard_exception", + "reason": "No mapping found for [dbGaP_phs] in order to sort on", + "index": "datasets", + "index_uuid": "JHoru6szQ8G_-NtFJd--Bg" + } + ], + "type": "search_phase_execution_exception", + "reason": "all shards failed", + "phase": "query", + "grouped": true, + "failed_shards": [ + { + "shard": 0, + "index": "datasets", + "node": "YzqehJMeSham0G7I2LQdmw", + "reason": { + "type": "query_shard_exception", + "reason": "No mapping found for [dbGaP_phs] in order to sort on", + "index": "datasets", + "index_uuid": "JHoru6szQ8G_-NtFJd--Bg" + } + } + ] + }, + "status": 400 +}; diff --git a/Services/dataset.service.test.js b/Services/dataset.service.test.js index 4aa0e4e..d0d2ced 100644 --- a/Services/dataset.service.test.js +++ b/Services/dataset.service.test.js @@ -22,6 +22,7 @@ import { normalOptions, normalSearchText, normalOpensearchResults, + errorOpensearchResults, } from './dataset.service.test.fixtures.js'; beforeEach(() => { @@ -83,4 +84,14 @@ describe('search', () => { const resultEmpty = await datasetService.search(normalSearchText, normalFilters, {}); expect(resultNull).toEqual(resultEmpty); }); + + it('should handle Opensearch error response', async () => { + const error = new Error("Test Opensearch failure"); + let result; + error.body = errorOpensearchResults; + vi.spyOn(elasticsearch, "searchWithAggregations").mockRejectedValue(error); + result = await datasetService.search(); + expect(result).toHaveProperty('error'); + expect(result.error).toBeDefined(); + }); }); From 2adc92744d9866c93074518cad34917ca6a8ce3a Mon Sep 17 00:00:00 2001 From: Joon Lee Date: Mon, 9 Feb 2026 18:17:50 +0000 Subject: [PATCH 4/9] Fixed an export --- Utils/datasetFields.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utils/datasetFields.js b/Utils/datasetFields.js index 5e7e799..61588e1 100644 --- a/Utils/datasetFields.js +++ b/Utils/datasetFields.js @@ -94,7 +94,7 @@ const datasetFields = { }; module.exports = { - DATASET_DEFAULT_SORT_FIELD: 'dataset_title_sort', + DATASET_DEFAULT_SORT_FIELD, DATASET_SEARCH_FIELDS, DATASET_HIGHLIGHT_FIELDS, DATASET_RETURN_FIELDS, From 67a1c87a5b623607022c84803221814d1834fe49 Mon Sep 17 00:00:00 2001 From: Joon Lee Date: Mon, 9 Feb 2026 18:20:57 +0000 Subject: [PATCH 5/9] Fixed sort validation --- Controllers/dataset.controllers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Controllers/dataset.controllers.js b/Controllers/dataset.controllers.js index 22783ef..bfa76ff 100644 --- a/Controllers/dataset.controllers.js +++ b/Controllers/dataset.controllers.js @@ -37,7 +37,7 @@ const search = async (req, res) => { // sort.name = "Resource"; // sort.k = "data_resource_id"; // } - if (sort?.v && !['asc', 'desc'].includes(sort?.v)) { + if (!(sort?.v && ['asc', 'desc'].includes(sort.v))) { sort.v = 'asc'; } From c0b3f1bd44475cb88447361ea38591bd8e9c9f6b Mon Sep 17 00:00:00 2001 From: Joon Lee Date: Mon, 9 Feb 2026 18:25:34 +0000 Subject: [PATCH 6/9] Refined test --- Services/dataset.service.test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Services/dataset.service.test.js b/Services/dataset.service.test.js index d0d2ced..62dd20a 100644 --- a/Services/dataset.service.test.js +++ b/Services/dataset.service.test.js @@ -90,7 +90,8 @@ describe('search', () => { let result; error.body = errorOpensearchResults; vi.spyOn(elasticsearch, "searchWithAggregations").mockRejectedValue(error); - result = await datasetService.search(); + result = await datasetService.search(normalSearchText, normalFilters, normalOptions); + expect(elasticsearch.searchWithAggregations).toHaveBeenCalled(); expect(result).toHaveProperty('error'); expect(result.error).toBeDefined(); }); From 78a9fe2727439a625d38e7e6e0dddfbd134a91db Mon Sep 17 00:00:00 2001 From: Joon Lee Date: Mon, 9 Feb 2026 18:30:40 +0000 Subject: [PATCH 7/9] Refined error response --- Controllers/dataset.controllers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Controllers/dataset.controllers.js b/Controllers/dataset.controllers.js index bfa76ff..2c40778 100644 --- a/Controllers/dataset.controllers.js +++ b/Controllers/dataset.controllers.js @@ -48,8 +48,8 @@ const search = async (req, res) => { const searchResult = await datasetService.search(searchText, filters, options); if (searchResult.error) { - res.json({ - status:"error", + res.status(500).json({ + status: "error", aggs: 'all', data: {}, error: searchResult.error, From a4e20fdb94b9d246220705153e0b3116308c682a Mon Sep 17 00:00:00 2001 From: Joon Lee Date: Tue, 10 Feb 2026 18:38:24 +0000 Subject: [PATCH 8/9] Removed old code --- Services/queryGenerator.js | 1 - 1 file changed, 1 deletion(-) diff --git a/Services/queryGenerator.js b/Services/queryGenerator.js index d14ab34..e3f9e69 100644 --- a/Services/queryGenerator.js +++ b/Services/queryGenerator.js @@ -343,7 +343,6 @@ queryGenerator.getDatasetFiltersQuery = (searchText, searchFilters, excludedFiel // Customize search query body.aggs = {}; body.size = 0; - delete query.highlight; // Aggregate on the target field body.aggs[excludedField] = { From 426f21dfd5ad952d277ea2a865869cd1bc20253f Mon Sep 17 00:00:00 2001 From: Joon Lee Date: Tue, 10 Feb 2026 18:54:29 +0000 Subject: [PATCH 9/9] Improved param validation for /search and /filters --- Controllers/dataset.controllers.js | 29 ++++++++++----- Services/dataset.service.js | 28 +++++++++++++-- Utils/datasetFields.js | 1 - Utils/params.js | 58 ++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 Utils/params.js diff --git a/Controllers/dataset.controllers.js b/Controllers/dataset.controllers.js index 2c40778..bb4704a 100644 --- a/Controllers/dataset.controllers.js +++ b/Controllers/dataset.controllers.js @@ -4,16 +4,17 @@ const config = require('../Config'); const path = require('path'); const { Parser } = require('json2csv'); const { DATASET_DEFAULT_SORT_FIELD, datasetFields } = require('../Utils/datasetFields'); +const { getObjectParam, getStringParam } = require('../Utils/params'); const datasetService = require('../Services/dataset.service'); const search = async (req, res) => { - const body = req.body; + const body = getObjectParam(req, 'body'); const data = {}; - const filters = body.filters && typeof body.filters === 'object' && !Array.isArray(body.filters) ? body.filters : {}; + const filters = getObjectParam(body, 'filters'); const options = {}; - const pageInfo = body.pageInfo ?? {page: 1, pageSize: 10}; - const searchText = body.search_text && typeof body.search_text === 'string' ? body.search_text.trim() : ''; - const sort = body.sort ?? {k: DATASET_DEFAULT_SORT_FIELD, v: 'asc'}; + const pageInfo = getObjectParam(body, 'pageInfo', {page: 1, pageSize: 10}); + const searchText = getStringParam(body, 'search_text'); + const sort = getObjectParam(body, 'sort', {k: DATASET_DEFAULT_SORT_FIELD, v: 'asc'}); if (pageInfo.page !== parseInt(pageInfo.page, 10) || pageInfo.page <= 0) { pageInfo.page = 1; @@ -47,6 +48,8 @@ const search = async (req, res) => { data.pageInfo = options.pageInfo; const searchResult = await datasetService.search(searchText, filters, options); + + // Error response if there's an error if (searchResult.error) { res.status(500).json({ status: "error", @@ -128,12 +131,22 @@ const getById = async (req, res) => { }; const getFilters = async (req, res) => { - const body = req.body; - const searchText = body.search_text?.trim() ?? ''; - const searchFilters = body.filters ?? {}; + const body = getObjectParam(req, 'body'); + const searchText = getStringParam(body, 'search_text'); + const searchFilters = getObjectParam(body, 'filters'); const filters = await datasetService.getFilters(searchText, searchFilters); + // Error response if there's an error + if (filters.error) { + res.status(500).json({ + status: "error", + data: {}, + error: filters.error, + }); + return; + } + res.json({status: 'success', data: filters}); }; diff --git a/Services/dataset.service.js b/Services/dataset.service.js index f1508ab..4050175 100644 --- a/Services/dataset.service.js +++ b/Services/dataset.service.js @@ -6,7 +6,6 @@ const mysql = require('../Components/mysql'); const queryGenerator = require('./queryGenerator'); const cacheKeyGenerator = require('./cacheKeyGenerator'); const utils = require('../Utils'); - const { DATASET_RETURN_FIELDS } = require('../Utils/datasetFields.js'); const FACET_FILTERS = [ 'dataset_source_repo', @@ -143,11 +142,36 @@ const getFilters = async (searchText, searchFilters) => { filters = {}; + // Check searchText type + if (searchText && typeof searchText !== 'string') { + return filters; + } + + // Check filters type + if (searchFilters && (typeof searchFilters !== 'object' || Array.isArray(searchFilters))) { + return filters; + } + + // Format the search text + if (searchText) { + const sanitizedSearchText = searchText.replace(/[^a-zA-Z0-9]+/g, ' '); // Ignore special characters + searchableText = utils.getSearchableText(sanitizedSearchText); + } + // Must obtain counts for each filter as if the filter were not applied await Promise.all(FACET_FILTERS.map(async (filterName) => { // Obtain counts from Opensearch + let filtersResponse; const query = queryGenerator.getDatasetFiltersQuery(searchText, searchFilters, filterName); - const filtersResponse = await elasticsearch.searchWithAggregations(config.indexDS, query); + + try { + filtersResponse = await elasticsearch.searchWithAggregations(config.indexDS, query); + } catch (error) { + logger.error(`Error searching datasets: ${error}`); + return { + error: error?.body?.error?.root_cause ? JSON.stringify(error.body.error.root_cause).replace(/\\n/g, '') : error.message, + }; + } // Extract counts from response filters[filterName] = filtersResponse.aggs[filterName].buckets.map((bucket) => ({ diff --git a/Utils/datasetFields.js b/Utils/datasetFields.js index 61588e1..8537e33 100644 --- a/Utils/datasetFields.js +++ b/Utils/datasetFields.js @@ -100,4 +100,3 @@ module.exports = { DATASET_RETURN_FIELDS, datasetFields, }; - diff --git a/Utils/params.js b/Utils/params.js new file mode 100644 index 0000000..bef4e2d --- /dev/null +++ b/Utils/params.js @@ -0,0 +1,58 @@ +/** + * Gets an object parameter from an object + * @param {Object} obj The object to get the parameter from + * @param {String} propName The name of the parameter to get + * @param {Object} defaultReturn The default value to return if there are problems + * @returns {Object} The parameter value + */ +const getObjectParam = (obj, propName, defaultReturn = {}) => { + let val; + + if (typeof obj !== 'object' || Array.isArray(obj)) { + return defaultReturn; + } + + if (!obj[propName]) { + return defaultReturn; + } + + if (typeof obj[propName] !== 'object' || Array.isArray(obj[propName])) { + return defaultReturn; + } + + val = obj[propName]; + + return val; +}; + +/** + * Gets a string parameter from an object + * @param {Object} obj The object to get the parameter from + * @param {String} propName The name of the parameter to get + * @param {String} defaultReturn The default value to return if there are problems + * @returns {String} The parameter value + */ +const getStringParam = (obj, propName, defaultReturn = '') => { + let val = ''; + + if (typeof obj !== 'object' || Array.isArray(obj)) { + return defaultReturn; + } + + if (!obj[propName]) { + return defaultReturn; + } + + if (typeof obj[propName] !== 'string') { + return defaultReturn; + } + + val = obj[propName].trim(); + + return val; +}; + +module.exports = { + getObjectParam, + getStringParam, +}; \ No newline at end of file