OpenSearchのLLM-as-a-Judgeを日本語の商品検索で動かして、人手ラベルと比べてみた

ARG OPENSEARCH_VERSION=3.9.0
FROM opensearchproject/opensearch:${OPENSEARCH_VERSION}
RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-kuromoji
# 3.9.0のSRWプラグインにはslf4j-apiが入っておらず、LLM_JUDGMENTでLLMの応答を受け取ると
# NoClassDefFoundError: org/slf4j/LoggerFactoryでOpenSearchが落ちる。ML Commonsのものをコピーして補う
RUN cp /usr/share/opensearch/plugins/opensearch-ml/slf4j-api-1.7.36.jar \
       /usr/share/opensearch/plugins/opensearch-search-relevance/

name: srw-esci-ja

services:
  opensearch:
    build:
      context: ./opensearch
      args:
        OPENSEARCH_VERSION: 3.9.0
    image: srw-esci-ja/opensearch-kuromoji:3.9.0
    container_name: srw-opensearch
    cpus: 4 # LLM_JUDGMENTが同時にLLMを呼ぶ数はCPUの数で決まるので、APIの回数制限に当たらないよう絞る
    environment:
      - discovery.type=single-node
      - bootstrap.memory_lock=true
      - OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g
      - DISABLE_INSTALL_DEMO_CONFIG=true # 動作確認用なのでdisableにする
      - DISABLE_SECURITY_PLUGIN=true # 動作確認用なのでdisableにする
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    volumes:
      - opensearch-data:/usr/share/opensearch/data
    ports:
      - 9200:9200

  dashboards:
    image: opensearchproject/opensearch-dashboards:3.9.0
    container_name: srw-dashboards
    environment:
      - OPENSEARCH_HOSTS=["http://opensearch:9200"]
      - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true # 動作確認用なのでdisableにする
    ports:
      - 5601:5601
    depends_on:
      - opensearch

volumes:
  opensearch-data:

$ docker compose -f docker/docker-compose.yml up -d --build

$ curl -L -O https://github.com/amazon-science/esci-data/raw/refs/heads/main/shopping_queries_dataset/shopping_queries_dataset_examples.parquet
$ curl -L -O https://github.com/amazon-science/esci-data/raw/refs/heads/main/shopping_queries_dataset/shopping_queries_dataset_products.parquet

PUT products_kuromoji
{
  "mappings": {
    "properties": {
      "product_id": { "type": "keyword" },
      "product_title": { "type": "text", "analyzer": "kuromoji" },
      "product_brand": { "type": "text", "analyzer": "kuromoji" },
      "product_bullet_point": { "type": "text", "analyzer": "kuromoji" },
      "product_description": { "type": "text", "analyzer": "kuromoji" },
      "candidate_queries": { "type": "keyword" } // どのクエリの候補か
    }
  }
}

PUT _cluster/settings
{
  "persistent": {
    "plugins.ml_commons.trusted_connector_endpoints_regex": [
      "^https://generativelanguage\\.googleapis\\.com/.*$"
    ],
    "plugins.ml_commons.only_run_on_ml_node": false
  }
}

POST _plugins/_ml/connectors/_create
{
  "name": "Gemini gemini-3.5-flash-lite",
  "version": "1",
  "protocol": "http",
  "parameters": { "model": "gemini-3.5-flash-lite" },
  "credential": { "gemini_key": "<API キー>" },
  "client_config": {
    "read_timeout": 180,
    "max_retry_times": 5,
    "retry_backoff_policy": "exponential_full_jitter",
    "retry_backoff_millis": 5000,
    "retry_timeout_seconds": 300,
    "connection_timeout": 30
  },
  "actions": [
    {
      "action_type": "predict",
      "method": "POST",
      "url": "https://generativelanguage.googleapis.com/v1beta/models/${parameters.model}:generateContent",
      "headers": {
        "Content-Type": "application/json",
        "x-goog-api-key": "${credential.gemini_key}"
      },
      "request_body": "{\"systemInstruction\":{\"parts\":[{\"text\":\"${parameters.system_prompt}\"}]},\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"${parameters.user_prompt}\"}]}]}",
      "post_process_function": "def text = params.candidates[0].content.parts[0].text.replace('\\\\\"', '\"'); return '{\"name\":\"response\",\"dataAsMap\":{\"response\":\"' + escape(text) + '\"}}'"
    }
  ]
}

POST _plugins/_ml/model_groups/_register
{ "name": "srw_llm_judgment" }

POST _plugins/_ml/models/_register?deploy=true
{
  "name": "Gemini gemini-3.5-flash-lite",
  "function_name": "remote",
  "model_group_id": "<model_group_id>",
  "connector_id": "<connector_id>"
}

PUT _plugins/_search_relevance/query_sets
{
  "name": "handson",
  "sampling": "manual",
  "querySetQueries": [{ "queryText": "電源なしスピーカー" }]
}

PUT _plugins/_search_relevance/search_configurations
{
  "name": "bm25-kuromoji",
  "index": "products_kuromoji",
  "query": "{\"query\":{\"multi_match\":{\"query\":\"%SearchText%\",\"fields\":[\"product_title^2\",\"product_brand\",\"product_bullet_point\",\"product_description\"]}}}"
}

PUT _plugins/_search_relevance/judgments
{
  "name": "handson-score",
  "type": "LLM_JUDGMENT",
  "modelId": "<model_id>",
  "querySetId": "<query_set_id>",
  "searchConfigurationList": ["<search_configuration_id>"],
  "size": 5, // 検索結果の上位何件を採点するか
  "contextFields": ["product_title", "product_brand", "product_bullet_point"], // LLMに渡すフィールド
  "llmJudgmentRatingType": "SCORE0_1" // 0〜1の段階評価
}

GET _plugins/_search_relevance/judgments/<judgment_id>

PUT _plugins/_search_relevance/search_configurations
{
  "name": "esci-candidates",
  "index": "products_kuromoji",
  "query": "{\"query\":{\"term\":{\"candidate_queries\":\"%SearchText%\"}}}"
}

PUT _plugins/_search_relevance/judgments
{
  "name": "full-score01",
  "type": "LLM_JUDGMENT",
  "modelId": "<model_id>",
  "querySetId": "<query_set_id>",
  "searchConfigurationList": ["<esci-candidates の ID>"],
  "size": 40, // 候補の最大件数
  "tokenLimit": 50000, // 1リクエストのトークン上限で、分割を減らすため大きめに設定
  "contextFields": ["product_title", "product_brand", "product_bullet_point"],
  "llmJudgmentRatingType": "SCORE0_1"
}