쿼리 확장 + Query Understanding — 동의어·오타 교정·snippet 고도화

// 1. 새 디렉토리에 전체 색인
Path newIndexPath = Paths.get("/data/lucene/wiki-index-" + version);
Directory newDir = MMapDirectory.open(newIndexPath);
IndexWriterConfig config = new IndexWriterConfig(analyzer);
IndexWriter newWriter = new IndexWriter(newDir, config);

// 2. DB 전체 스캔 + 배치 색인 (2000건 단위)
try (Stream<Post> posts = postRepository.streamAll()) {
    List<Document> batch = new ArrayList<>(2000);
    posts.forEach(post -> {
        batch.add(toDocument(post));
        if (batch.size() >= 2000) {
            newWriter.addDocuments(batch);
            batch.clear();
        }
    });
    if (!batch.isEmpty()) newWriter.addDocuments(batch);
}
newWriter.commit();
newWriter.close();

// 3. 심볼릭 링크 원자적 교체
Path symlink = Paths.get("/data/lucene/wiki-index");
Path tempLink = Paths.get("/data/lucene/wiki-index-tmp");
Files.createSymbolicLink(tempLink, newIndexPath);
Files.move(tempLink, symlink,
    StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);

// 4. SearcherManager를 새 Directory로 재생성
// MMapDirectory는 파일을 메모리 매핑하므로, 심볼릭 링크 교체만으로는 부족
// SearcherManager를 닫고 새 Directory로 다시 생성해야 함
searcherManager.close();
Directory currentDir = MMapDirectory.open(symlink);
searcherManager = new SearcherManager(currentDir, null);

private final AtomicBoolean fullReindexInProgress = new AtomicBoolean(false);

public void incrementalIndex(Post post) throws IOException {
    if (fullReindexInProgress.get()) {
        log.warn("Full reindex in progress, skipping incremental for post={}",
                 post.getId());
        // CDC 이벤트는 Kafka에 남아있으므로, 재색인 완료 후 자동 재처리
        return;
    }
    writer.updateDocument(
        new Term("id", String.valueOf(post.getId())), toDocument(post));
}

implementation 'org.apache.lucene:lucene-highlighter:10.3.2'
implementation 'org.apache.lucene:lucene-suggest:10.3.2'    // DirectSpellChecker

// 방안 A: content를 Store.YES로 변경 (단순하지만 인덱스 100GB+)
doc.add(new TextField("content", post.getContent(), Field.Store.YES));

// 방안 B: 별도 snippet_source 저장 (앞 500자만, 인덱스 크기 최소화) — 선택
String snippetSource = post.getContent().substring(
    0, Math.min(post.getContent().length(), 500));
doc.add(new StoredField("snippetSource", snippetSource));
// content는 Store.NO 유지 (검색용만)

UnifiedHighlighter highlighter = UnifiedHighlighter.builder(searcher, analyzer)
    .withFieldMatcher(field -> "snippetSource".equals(field))
    .build();
String[] snippets = highlighter.highlight("snippetSource", query, topDocs, 1);

// snippet이 없으면 (500자 밖에 검색어가 있는 경우) DB fallback
for (int i = 0; i < results.size(); i++) {
    if (snippets[i] != null && !snippets[i].isBlank()) {
        results.get(i).setSnippet(snippets[i]);
    }
    // else: PostSearchResponse.createSnippet()으로 fallback (현재 방식)
}

CREATE TABLE synonyms (
    id BIGINT AUTO_INCREMENT PRIMARY KEY,
    term VARCHAR(100) NOT NULL,
    synonym VARCHAR(100) NOT NULL,
    weight DOUBLE DEFAULT 1.0,
    INDEX idx_term (term),
    INDEX idx_synonym (synonym)
);

-- 수동 등록
INSERT INTO synonyms (term, synonym, weight) VALUES
    ('AI', '인공지능', 1.0),
    ('인공지능', 'AI', 1.0),
    ('ML', '머신러닝', 1.0),
    ('머신러닝', 'ML', 1.0),
    ('DB', '데이터베이스', 1.0),
    ('데이터베이스', 'DB', 1.0);

SELECT redirect_title AS term, title AS synonym, 0.8 AS weight
FROM posts
WHERE redirect_title IS NOT NULL;
-- 예: "인공 지능" → "인공지능", "AI" → "인공지능"

@Service
public class QueryExpansionService {

    private final SynonymRepository synonymRepository;

    /**
     * 원래 쿼리의 각 term에 대해 동의어를 찾아 확장한다.
     * term당 최대 3개 동의어로 제한하여 쿼리 폭발을 방지한다.
     */
    public List<ExpandedTerm> expandQuery(List<String> originalTerms) {
        List<ExpandedTerm> expanded = new ArrayList<>();

        for (String term : originalTerms) {
            // 원래 term은 boost=1.0
            expanded.add(new ExpandedTerm(term, 1.0, true));

            // 동의어는 가중치 적용, 최대 3개
            List<Synonym> synonyms = synonymRepository.findByTerm(term);
            synonyms.stream()
                .limit(3)
                .forEach(syn -> expanded.add(
                    new ExpandedTerm(syn.getSynonym(), syn.getWeight(), false)
                ));
        }
        return expanded;
    }
}

// 기존: parser.parse("AI")
//   → TermQuery("ai")

// 동의어 확장 후: "AI" + "인공지능" (weight=1.0)
//   → BooleanQuery(
//       TermQuery("ai")^1.0,           SHOULD
//       TermQuery("인공지능")^1.0       SHOULD
//     )

@Service
public class SpellCheckService {

    private final SearcherManager searcherManager;

    /**
     * Lucene 인덱스의 term dictionary를 사전으로 활용하여
     * 편집 거리(Damerau-Levenshtein) 기반 오타 교정을 수행한다.
     *
     * DirectSpellChecker는 별도 사전 구축 없이, 인덱스가 곧 사전이다.
     *
     * 한국어 한계:
     * - 음절 단위 비교이므로 "컴퓨텨"→"컴퓨터"(편집 거리 1)는 잡히지만,
     *   "프로그래링"→"프로그래밍"(ㅁ 누락)은 자모 레벨에서 1글자 차이인데
     *   음절 레벨에서도 편집 거리 1이라 잡힘.
     * - 진짜 문제: 인덱스에 해당 term이 없으면 후보를 못 찾음.
     *   Nori가 복합어를 분해하므로 인덱스 term이 원형과 다를 수 있다.
     * - 이 한계는 검색 로그 기반 "Did you mean?"으로 보강한다.
     */
    public Optional<String> suggestCorrection(String query) throws IOException {
        IndexSearcher searcher = searcherManager.acquire();
        try {
            DirectSpellChecker spellChecker = new DirectSpellChecker();
            spellChecker.setMaxEdits(2);           // 최대 편집 거리 2
            spellChecker.setMinPrefix(1);           // 첫 글자는 일치해야 함
            spellChecker.setMinQueryLength(2);      // 2글자 미만은 교정 안 함

            String[] tokens = tokenize(query);
            List<String> corrected = new ArrayList<>();
            boolean hasCorrected = false;

            for (String token : tokens) {
                SuggestWord[] suggestions = spellChecker.suggestSimilar(
                    new Term("title", token),
                    1,                              // 최대 1개 제안
                    searcher.getIndexReader()
                );

                if (suggestions.length > 0) {
                    corrected.add(suggestions[0].string);
                    hasCorrected = true;
                } else {
                    corrected.add(token);
                }
            }

            return hasCorrected
                ? Optional.of(String.join(" ", corrected))
                : Optional.empty();
        } finally {
            searcherManager.release(searcher);
        }
    }
}

# userdict_ko.txt — Nori 사용자 사전
운동화
에어맥스
나이키
인공지능
머신러닝
데이터베이스

// 1. full index into a fresh directory
Path newIndexPath = Paths.get("/data/lucene/wiki-index-" + version);
Directory newDir = MMapDirectory.open(newIndexPath);
IndexWriterConfig config = new IndexWriterConfig(analyzer);
IndexWriter newWriter = new IndexWriter(newDir, config);

// 2. full DB scan + batch indexing (2000-doc batches)
try (Stream<Post> posts = postRepository.streamAll()) {
    List<Document> batch = new ArrayList<>(2000);
    posts.forEach(post -> {
        batch.add(toDocument(post));
        if (batch.size() >= 2000) {
            newWriter.addDocuments(batch);
            batch.clear();
        }
    });
    if (!batch.isEmpty()) newWriter.addDocuments(batch);
}
newWriter.commit();
newWriter.close();

// 3. atomic symlink swap
Path symlink = Paths.get("/data/lucene/wiki-index");
Path tempLink = Paths.get("/data/lucene/wiki-index-tmp");
Files.createSymbolicLink(tempLink, newIndexPath);
Files.move(tempLink, symlink,
    StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);

// 4. recreate SearcherManager on the new Directory
// MMapDirectory maps files into memory; symlink swap alone is not enough
// must close SearcherManager and recreate on the new Directory
searcherManager.close();
Directory currentDir = MMapDirectory.open(symlink);
searcherManager = new SearcherManager(currentDir, null);

private final AtomicBoolean fullReindexInProgress = new AtomicBoolean(false);

public void incrementalIndex(Post post) throws IOException {
    if (fullReindexInProgress.get()) {
        log.warn("Full reindex in progress, skipping incremental for post={}",
                 post.getId());
        // CDC events remain in Kafka — auto-reprocessed after reindex
        return;
    }
    writer.updateDocument(
        new Term("id", String.valueOf(post.getId())), toDocument(post));
}

implementation 'org.apache.lucene:lucene-highlighter:10.3.2'
implementation 'org.apache.lucene:lucene-suggest:10.3.2'    // DirectSpellChecker

// Option A: change content to Store.YES (simple but index 100GB+)
doc.add(new TextField("content", post.getContent(), Field.Store.YES));

// Option B: separate snippet_source store (first 500 chars, minimizes growth) — chosen
String snippetSource = post.getContent().substring(
    0, Math.min(post.getContent().length(), 500));
doc.add(new StoredField("snippetSource", snippetSource));
// content stays Store.NO (search only)

UnifiedHighlighter highlighter = UnifiedHighlighter.builder(searcher, analyzer)
    .withFieldMatcher(field -> "snippetSource".equals(field))
    .build();
String[] snippets = highlighter.highlight("snippetSource", query, topDocs, 1);

// if no snippet (query lies beyond first 500 chars), fall back to DB
for (int i = 0; i < results.size(); i++) {
    if (snippets[i] != null && !snippets[i].isBlank()) {
        results.get(i).setSnippet(snippets[i]);
    }
    // else: fallback via PostSearchResponse.createSnippet() (current method)
}

CREATE TABLE synonyms (
    id BIGINT AUTO_INCREMENT PRIMARY KEY,
    term VARCHAR(100) NOT NULL,
    synonym VARCHAR(100) NOT NULL,
    weight DOUBLE DEFAULT 1.0,
    INDEX idx_term (term),
    INDEX idx_synonym (synonym)
);

-- manual entries
INSERT INTO synonyms (term, synonym, weight) VALUES
    ('AI', '인공지능', 1.0),
    ('인공지능', 'AI', 1.0),
    ('ML', '머신러닝', 1.0),
    ('머신러닝', 'ML', 1.0),
    ('DB', '데이터베이스', 1.0),
    ('데이터베이스', 'DB', 1.0);

SELECT redirect_title AS term, title AS synonym, 0.8 AS weight
FROM posts
WHERE redirect_title IS NOT NULL;
-- e.g., "인공 지능" → "인공지능", "AI" → "인공지능"

@Service
public class QueryExpansionService {

    private final SynonymRepository synonymRepository;

    /**
     * Find synonyms for each term in the original query and expand.
     * Cap synonyms per term at 3 to prevent query explosion.
     */
    public List<ExpandedTerm> expandQuery(List<String> originalTerms) {
        List<ExpandedTerm> expanded = new ArrayList<>();

        for (String term : originalTerms) {
            // original term has boost=1.0
            expanded.add(new ExpandedTerm(term, 1.0, true));

            // synonyms: weighted, max 3
            List<Synonym> synonyms = synonymRepository.findByTerm(term);
            synonyms.stream()
                .limit(3)
                .forEach(syn -> expanded.add(
                    new ExpandedTerm(syn.getSynonym(), syn.getWeight(), false)
                ));
        }
        return expanded;
    }
}

// Before: parser.parse("AI")
//   → TermQuery("ai")

// After expansion: "AI" + "인공지능" (weight=1.0)
//   → BooleanQuery(
//       TermQuery("ai")^1.0,           SHOULD
//       TermQuery("인공지능")^1.0       SHOULD
//     )

@Service
public class SpellCheckService {

    private final SearcherManager searcherManager;

    /**
     * Use the Lucene index's term dictionary as the spell-check dictionary.
     * Edit-distance (Damerau-Levenshtein) corrections.
     *
     * DirectSpellChecker needs no separate dictionary build — the index is the dictionary.
     *
     * Korean limits:
     * - Comparison is syllable-level: "컴퓨텨"→"컴퓨터" (edit distance 1) is caught.
     *   "프로그래링"→"프로그래밍" (missing ㅁ) is jamo-level 1, but also syllable-level 1, so caught.
     * - Real issue: if the index has no such term, no candidate.
     *   Nori decomposes compounds, so the indexed term may differ from the original.
     * - Reinforce later with log-based "Did you mean?".
     */
    public Optional<String> suggestCorrection(String query) throws IOException {
        IndexSearcher searcher = searcherManager.acquire();
        try {
            DirectSpellChecker spellChecker = new DirectSpellChecker();
            spellChecker.setMaxEdits(2);           // max edit distance 2
            spellChecker.setMinPrefix(1);           // first char must match
            spellChecker.setMinQueryLength(2);      // skip <2 chars

            String[] tokens = tokenize(query);
            List<String> corrected = new ArrayList<>();
            boolean hasCorrected = false;

            for (String token : tokens) {
                SuggestWord[] suggestions = spellChecker.suggestSimilar(
                    new Term("title", token),
                    1,                              // 1 suggestion max
                    searcher.getIndexReader()
                );

                if (suggestions.length > 0) {
                    corrected.add(suggestions[0].string);
                    hasCorrected = true;
                } else {
                    corrected.add(token);
                }
            }

            return hasCorrected
                ? Optional.of(String.join(" ", corrected))
                : Optional.empty();
        } finally {
            searcherManager.release(searcher);
        }
    }
}

# userdict_ko.txt — Nori user dictionary
운동화
에어맥스
나이키
인공지능
머신러닝
데이터베이스