Add support for query tokenization parameters

app/controllers/search_controller.rb

@@ -226,7 +226,8 @@ def query_primo(per_page, offset)
   end
 
   def execute_geospatial_query(query)
-    query = query.except('queryMode')
+    query = query.except('queryMode', 'semanticDropBoostThreshold', 'semanticMustBoostThreshold',
+                         'semanticShortQueryMaxTokens')
 
     if query['geobox'] == 'true' && query[:geodistance] == 'true'
       TimdexBase::Client.query(TimdexSearch::AllQuery, variables: query)

app/helpers/application_helper.rb

@@ -20,7 +20,8 @@ def index_page_title
   def results_page_title(query, character_limit = 50)
     return index_page_title unless query.present?
 
-    ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode]
+    ignored_terms = %i[page advanced geobox geodistance booleanType tab queryMode semanticDropBoostThreshold
+                       semanticMustBoostThreshold semanticShortQueryMaxTokens]
     terms = query.reject { |term| ignored_terms.include? term }.values.join(' ')
     terms = "#{terms.first(character_limit)}..." if terms.length > character_limit
     "#{terms} | #{index_page_title}"

app/models/enhancer.rb

@@ -1,7 +1,8 @@
 class Enhancer
   attr_accessor :enhanced_query
 
-  QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title queryMode].freeze
+  QUERY_PARAMS = %i[q citation contentType contributors fundingInformation identifiers locations subjects title
+                    queryMode semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze
   FILTER_PARAMS = %i[accessToFilesFilter contentTypeFilter contributorsFilter formatFilter languagesFilter
                      literaryFormFilter placesFilter sourceFilter subjectsFilter].freeze
   GEO_PARAMS = %i[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude

app/models/query_builder.rb

@@ -7,6 +7,7 @@ class QueryBuilder
   GEO_PARAMS = %w[geoboxMinLongitude geoboxMinLatitude geoboxMaxLongitude geoboxMaxLatitude geodistanceLatitude
                   geodistanceLongitude geodistanceDistance].freeze
   VALID_QUERY_MODES = %w[keyword semantic hybrid].freeze
+  TOKENIZATION_PARAMS = %w[semanticDropBoostThreshold semanticMustBoostThreshold semanticShortQueryMaxTokens].freeze
 
   def initialize(enhanced_query)
     @query = {}
@@ -22,6 +23,7 @@ def initialize(enhanced_query)
     extract_geosearch(enhanced_query)
     extract_filters(enhanced_query)
     evaluate_query_mode(enhanced_query)
+    extract_tokenization_params(enhanced_query)
     @query['index'] = ENV.fetch('TIMDEX_INDEX', nil)
     @query['booleanType'] = enhanced_query[:booleanType]
     @query.compact!
@@ -59,9 +61,20 @@ def extract_filters(enhanced_query)
     end
   end
 
-  # The GraphQL API requires that lat/long in geospatial fields be floats
-  def coerce_to_float?(geo_param)
-    geo_param.to_s.include?('Longitude') || geo_param.to_s.include?('Latitude')
+  # We treat the tokenization parameters separately because we need to ensure that floats and integers are formatted
+  # correctly.
+  def extract_tokenization_params(enhanced_query)
+    TOKENIZATION_PARAMS.each do |tp|
+      next unless enhanced_query[tp.to_sym].present?
+
+      @query[tp] = coerce_to_float?(tp) ? enhanced_query[tp.to_sym]&.strip.to_f : enhanced_query[tp.to_sym]&.strip.to_i
+    end
+  end
+
+  # The GraphQL API requires that some parameters - lat/long in geospatial fields and boost thresholds for tuning - be
+  # floats.
+  def coerce_to_float?(param)
+    param.to_s.include?('Longitude') || param.to_s.include?('Latitude') || param.to_s.include?('BoostThreshold')
   end
 
   # Determine the query mode from URL parameter or config, with fallback to 'keyword'

app/models/timdex_search.rb

@@ -28,6 +28,9 @@ class TimdexSearch < TimdexBase
       $sourceFilter: [String!]
       $subjectsFilter: [String!]
       $useGlobalScoring: Boolean
+      $semanticDropBoostThreshold: Float
+      $semanticMustBoostThreshold: Float
+      $semanticShortQueryMaxTokens: Int
     ) {
       search(
         searchterm: $q
@@ -54,6 +57,11 @@ class TimdexSearch < TimdexBase
         sourceFilter: $sourceFilter
         subjectsFilter: $subjectsFilter
         useGlobalScoring: $useGlobalScoring
+        tuningParametersInput: {
+          dropBoostThreshold: $semanticDropBoostThreshold
+          mustBoostThreshold: $semanticMustBoostThreshold
+          shortQueryMaxTokens: $semanticShortQueryMaxTokens
+        }
       ) {
         hits
         records {

config/schema/schema.json

@@ -1533,6 +1533,16 @@
                     }
                   },
                   "defaultValue": "null"
+                },
+                {
+                  "name": "tuningParametersInput",
+                  "description": "Experimental tuning parameters for semantic search. Not recommended for use.",
+                  "type": {
+                    "kind": "INPUT_OBJECT",
+                    "name": "TuningParametersInput",
+                    "ofType": null
+                  },
+                  "defaultValue": "null"
                 }
               ],
               "type": {
@@ -2606,6 +2616,47 @@
           "enumValues": null,
           "possibleTypes": null
         },
+        {
+          "kind": "INPUT_OBJECT",
+          "name": "TuningParametersInput",
+          "description": "Experimental tuning parameters for semantic search. Not recommended for use.",
+          "fields": null,
+          "inputFields": [
+            {
+              "name": "mustBoostThreshold",
+              "description": "Not recommended for use. Range: 0.0 to 1.0",
+              "type": {
+                "kind": "SCALAR",
+                "name": "Float",
+                "ofType": null
+              },
+              "defaultValue": "null"
+            },
+            {
+              "name": "dropBoostThreshold",
+              "description": "Not recommended for use. Range: 0.0 to 1.0",
+              "type": {
+                "kind": "SCALAR",
+                "name": "Float",
+                "ofType": null
+              },
+              "defaultValue": "null"
+            },
+            {
+              "name": "shortQueryMaxTokens",
+              "description": "Not recommended for use. Must be greater than 0",
+              "type": {
+                "kind": "SCALAR",
+                "name": "Int",
+                "ofType": null
+              },
+              "defaultValue": "null"
+            }
+          ],
+          "interfaces": null,
+          "enumValues": null,
+          "possibleTypes": null
+        },
         {
           "kind": "OBJECT",
           "name": "Version",

test/controllers/search_controller_test.rb

@@ -1183,6 +1183,26 @@ def source_filter_count(controller)
     assert_select '.pagination-container .current', text: /21 - 40 of 800/
   end
 
+  # test 'results can include tuning parameters' do
+  #   query = 'optical networks in space'
+  #   must_default = 0.7
+  #   drop_default = 0.1
+  #   must_alt = 0.9
+  #   drop_alt = 0.4
+
+  #   VCR.use_cassette('default tuning for stock query') do
+  #     get "/results?q=#{query}&semanticMustBoostThreshold=#{must_default}&semanticDropBoostThreshold=#{drop_default}&tab=timdex"
+  #     assert_response :success
+  #   end
+
+  #   VCR.use_cassette('alternate tuning for stock query') do
+  #     get "/results?q=#{query}&semanticMustBoostThreshold=#{must_alt}&semanticDropBoostThreshold=#{drop_alt}&tab=timdex"
+  #     assert_response :success
+  #   end
+
+  #   # Assert result counts are different
+  # end
+
   test 'results can be returned in JSON format when env is set and valid token is provided' do
     secret_value = 'sooper_sekret'
     quepid_ua = 'Quepid/1.0 (Web Scraper)'

test/helpers/application_helper_test.rb

@@ -54,6 +54,12 @@ class ApplicationHelperTest < ActionView::TestCase
     assert_equal 'Search MIT Libraries', results_page_title(no_query)
   end
 
+  test 'results_page_title ignores tuning parameters' do
+    query = { q: 'National Parks Service', semanticDropBoostThreshold: '0.1', semanticMustBoostThreshold: '0.7',
+              semanticShortQueryMaxTokens: '5' }
+    assert_equal 'National Parks Service | Search MIT Libraries', results_page_title(query)
+  end
+
   test 'record_page_title includes record title' do
     record = { 'title' => 'The Waves' }
     assert_equal 'The Waves | Search MIT Libraries', record_page_title(record)

test/models/query_builder_test.rb

@@ -123,6 +123,7 @@ class QueryBuilderTest < ActiveSupport::TestCase
     assert_equal expected, QueryBuilder.new(search).query
   end
 
+  # Query mode behavior
   test 'query builder defaults to keyword queryMode' do
     search = { q: 'blah' }
     assert_equal('keyword', QueryBuilder.new(search).query['queryMode'])
@@ -185,4 +186,28 @@ class QueryBuilderTest < ActiveSupport::TestCase
     search = { q: 'blah', queryMode: '  semantic  ' }
     assert_equal('semantic', QueryBuilder.new(search).query['queryMode'])
   end
+
+  # Tuning parameter behavior
+  test 'query builder handles tuning parameters' do
+    expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX',
+                 'semanticDropBoostThreshold' => 0.1, 'semanticMustBoostThreshold' => 0.7,
+                 'semanticShortQueryMaxTokens' => 5 }
+    search = { q: 'blah', semanticDropBoostThreshold: '0.1', semanticMustBoostThreshold: '0.7',
+               semanticShortQueryMaxTokens: '5' }
+    assert_equal(expected, QueryBuilder.new(search).query)
+  end
+
+  test 'query builder converts semanticShortQueryMaxTokens to integer' do
+    expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX',
+                 'semanticShortQueryMaxTokens' => 5 }
+    search = { q: 'blah', semanticShortQueryMaxTokens: '5.0' }
+    assert_equal(expected, QueryBuilder.new(search).query)
+  end
+
+  test 'query builder converts drop and boost thresholds to floats' do
+    expected = { 'from' => '0', 'q' => 'blah', 'queryMode' => 'keyword', 'index' => 'FAKE_TIMDEX_INDEX',
+                 'semanticDropBoostThreshold' => 0.0, 'semanticMustBoostThreshold' => 1.0 }
+    search = { q: 'blah', semanticDropBoostThreshold: '0', semanticMustBoostThreshold: '1' }
+    assert_equal(expected, QueryBuilder.new(search).query)
+  end
 end