diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index 16e4dd5e..47a3eaaa 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -226,7 +226,8 @@ def query_primo(per_page, offset) end def execute_geospatial_query(query) - query = query.except('queryMode') + query = query.except('queryMode', 'semanticDropBoostThreshold', 'semanticMustBoostThreshold', + 'semanticShortQueryMaxTokens') if query['geobox'] == 'true' && query[:geodistance] == 'true' TimdexBase::Client.query(TimdexSearch::AllQuery, variables: query) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index b085ceb8..9be0d2fc 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -20,7 +20,8 @@ def index_page_title def results_page_title(query, character_limit = 50) return index_page_title unless query.present? - ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode] + ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode semanticDropBoostThreshold + semanticMustBoostThreshold semanticShortQueryMaxTokens] terms = query.reject { |term| ignored_terms.include? term }.values.join(' ') terms = "#{terms.first(character_limit)}..." if terms.length > character_limit "#{terms} | #{index_page_title}" diff --git a/app/models/enhancer.rb b/app/models/enhancer.rb index b21099bf..7ee32a4a 100644 --- a/app/models/enhancer.rb +++ b/app/models/enhancer.rb @@ -1,7 +1,8 @@ class Enhancer attr_accessor :enhanced_query - QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title queryMode].freeze + QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title + queryMode semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze FILTER_PARAMS = %i[accessToFilesFilter contentTypeFilter contributorsFilter formatFilter languagesFilter literaryFormFilter placesFilter sourceFilter subjectsFilter].freeze GEO_PARAMS = %i[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude diff --git a/app/models/query_builder.rb b/app/models/query_builder.rb index 76491ca0..f7c4596c 100644 --- a/app/models/query_builder.rb +++ b/app/models/query_builder.rb @@ -7,6 +7,7 @@ class QueryBuilder GEO_PARAMS = %w[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude geodistanceLongitude geodistanceDistance].freeze VALID_QUERY_MODES = %w[keyword semantic hybrid].freeze + TOKENIZATION_PARAMS = %w[semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze def initialize(enhanced_query) @query = {} @@ -22,6 +23,7 @@ def initialize(enhanced_query) extract_geosearch(enhanced_query) extract_filters(enhanced_query) evaluate_query_mode(enhanced_query) + extract_tokenization_params(enhanced_query) @query['index'] = ENV.fetch('TIMDEX_INDEX', nil) @query['booleanType'] = enhanced_query[:booleanType] @query.compact! @@ -59,9 +61,20 @@ def extract_filters(enhanced_query) end end - # The GraphQL API requires that lat/long in geospatial fields be floats - def coerce_to_float?(geo_param) - geo_param.to_s.include?('Longitude') || geo_param.to_s.include?('Latitude') + # We treat the tokenization parameters separately because we need to ensure that floats and integers are formatted + # correctly. + def extract_tokenization_params(enhanced_query) + TOKENIZATION_PARAMS.each do |tp| + next unless enhanced_query[tp.to_sym].present? + + @query[tp] = coerce_to_float?(tp) ? enhanced_query[tp.to_sym]&.strip.to_f : enhanced_query[tp.to_sym]&.strip.to_i + end + end + + # The GraphQL API requires that some parameters - lat/long in geospatial fields and boost thresholds for tuning - be + # floats. + def coerce_to_float?(param) + param.to_s.include?('Longitude') || param.to_s.include?('Latitude') || param.to_s.include?('BoostThreshold') end # Determine the query mode from URL parameter or config, with fallback to 'keyword' diff --git a/app/models/timdex_search.rb b/app/models/timdex_search.rb index eb01f486..57ce31b6 100644 --- a/app/models/timdex_search.rb +++ b/app/models/timdex_search.rb @@ -28,6 +28,9 @@ class TimdexSearch < TimdexBase $sourceFilter: [String!] $subjectsFilter: [String!] $useGlobalScoring: Boolean + $semanticDropBoostThreshold: Float + $semanticMustBoostThreshold: Float + $semanticShortQueryMaxTokens: Int ) { search( searchterm: $q @@ -54,6 +57,11 @@ class TimdexSearch < TimdexBase sourceFilter: $sourceFilter subjectsFilter: $subjectsFilter useGlobalScoring: $useGlobalScoring + tuningParametersInput: { + dropBoostThreshold: $semanticDropBoostThreshold + mustBoostThreshold: $semanticMustBoostThreshold + shortQueryMaxTokens: $semanticShortQueryMaxTokens + } ) { hits records { diff --git a/config/schema/schema.json b/config/schema/schema.json index fba8727a..39866f1c 100644 --- a/config/schema/schema.json +++ b/config/schema/schema.json @@ -1533,6 +1533,16 @@ } }, "defaultValue": "null" + }, + { + "name": "tuningParametersInput", + "description": "Experimental tuning parameters for semantic search. Not recommended for use.", + "type": { + "kind": "INPUT_OBJECT", + "name": "TuningParametersInput", + "ofType": null + }, + "defaultValue": "null" } ], "type": { @@ -2606,6 +2616,47 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "INPUT_OBJECT", + "name": "TuningParametersInput", + "description": "Experimental tuning parameters for semantic search. Not recommended for use.", + "fields": null, + "inputFields": [ + { + "name": "mustBoostThreshold", + "description": "Not recommended for use. Range: 0.0 to 1.0", + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "defaultValue": "null" + }, + { + "name": "dropBoostThreshold", + "description": "Not recommended for use. Range: 0.0 to 1.0", + "type": { + "kind": "SCALAR", + "name": "Float", + "ofType": null + }, + "defaultValue": "null" + }, + { + "name": "shortQueryMaxTokens", + "description": "Not recommended for use. Must be greater than 0", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": "null" + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "Version", diff --git a/test/controllers/search_controller_test.rb b/test/controllers/search_controller_test.rb index 010c527e..c4af5ed5 100644 --- a/test/controllers/search_controller_test.rb +++ b/test/controllers/search_controller_test.rb @@ -1183,6 +1183,26 @@ def source_filter_count(controller) assert_select '.pagination-container .current', text: /21 - 40 of 800/ end + # test 'results can include tuning parameters' do + # query = 'optical networks in space' + # must_default = 0.7 + # drop_default = 0.1 + # must_alt = 0.9 + # drop_alt = 0.4 + + # VCR.use_cassette('default tuning for stock query') do + # get "/results?q=#{query}&semanticMustBoostThreshold=#{must_default}&semanticDropBoostThreshold=#{drop_default}&tab=timdex" + # assert_response :success + # end + + # VCR.use_cassette('alternate tuning for stock query') do + # get "/results?q=#{query}&semanticMustBoostThreshold=#{must_alt}&semanticDropBoostThreshold=#{drop_alt}&tab=timdex" + # assert_response :success + # end + + # # Assert result counts are different + # end + test 'results can be returned in JSON format when env is set and valid token is provided' do secret_value = 'sooper_sekret' quepid_ua = 'Quepid/1.0 (Web Scraper)' diff --git a/test/helpers/application_helper_test.rb b/test/helpers/application_helper_test.rb index a2a62000..74d6f28a 100644 --- a/test/helpers/application_helper_test.rb +++ b/test/helpers/application_helper_test.rb @@ -54,6 +54,12 @@ class ApplicationHelperTest < ActionView::TestCase assert_equal 'Search MIT Libraries', results_page_title(no_query) end + test 'results_page_title ignores tuning parameters' do + query = { q: 'National Parks Service', semanticDropBoostThreshold: '0.1', semanticMustBoostThreshold: '0.7', + semanticShortQueryMaxTokens: '5' } + assert_equal 'National Parks Service | Search MIT Libraries', results_page_title(query) + end + test 'record_page_title includes record title' do record = { 'title' => 'The Waves' } assert_equal 'The Waves | Search MIT Libraries', record_page_title(record) diff --git a/test/models/query_builder_test.rb b/test/models/query_builder_test.rb index 41b689fc..8f364b6b 100644 --- a/test/models/query_builder_test.rb +++ b/test/models/query_builder_test.rb @@ -123,6 +123,7 @@ class QueryBuilderTest < ActiveSupport::TestCase assert_equal expected, QueryBuilder.new(search).query end + # Query mode behavior test 'query builder defaults to keyword queryMode' do search = { q: 'blah' } assert_equal('keyword', QueryBuilder.new(search).query['queryMode']) @@ -185,4 +186,28 @@ class QueryBuilderTest < ActiveSupport::TestCase search = { q: 'blah', queryMode: ' semantic ' } assert_equal('semantic', QueryBuilder.new(search).query['queryMode']) end + + # Tuning parameter behavior + test 'query builder handles tuning parameters' do + expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX', + 'semanticDropBoostThreshold' => 0.1, 'semanticMustBoostThreshold' => 0.7, + 'semanticShortQueryMaxTokens' => 5 } + search = { q: 'blah', semanticDropBoostThreshold: '0.1', semanticMustBoostThreshold: '0.7', + semanticShortQueryMaxTokens: '5' } + assert_equal(expected, QueryBuilder.new(search).query) + end + + test 'query builder converts semanticShortQueryMaxTokens to integer' do + expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX', + 'semanticShortQueryMaxTokens' => 5 } + search = { q: 'blah', semanticShortQueryMaxTokens: '5.0' } + assert_equal(expected, QueryBuilder.new(search).query) + end + + test 'query builder converts drop and boost thresholds to floats' do + expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX', + 'semanticDropBoostThreshold' => 0.0, 'semanticMustBoostThreshold' => 1.0 } + search = { q: 'blah', semanticDropBoostThreshold: '0', semanticMustBoostThreshold: '1' } + assert_equal(expected, QueryBuilder.new(search).query) + end end