commit ccbd0797c583519bc67f5513ed8d5a37ce85c221 parent d542daa9f47c109b34807290fe27d2a590a5867c Author: andsy10 <andsy1016@gmail.com> Date: Fri, 16 Oct 2026 07:27:48 +0800 Add word-boundary-based Chinese word segmentation mode Diffstat:
| M | lisp/init-edit.el | | | 7 | ------- |
| M | lisp/init-segmentation.el | | | 108 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------- |
2 files changed, 87 insertions(+), 28 deletions(-)
diff --git a/lisp/init-edit.el b/lisp/init-edit.el @@ -16,13 +16,6 @@ :config (setq electric-pair-inhibit-predicate 'electric-pair-conservative-inhibit)) -;; subword movement -(my-use-package subword - :straight nil - :delight - :hook - (after-init . global-subword-mode)) - ;; repeat (my-use-package repeat :straight nil diff --git a/lisp/init-segmentation.el b/lisp/init-segmentation.el @@ -5,33 +5,99 @@ ;;; Code: ;; github@kanglmf/emacs-chinese-word-segmentation -(my-use-package cns - (file-exists-p my-cns-dir) +(my-use-package cns (file-exists-p my-cns-dir) :straight nil + :demand t :hook - (after-init . require-cns) - (after-init . global-cns-mode) + (after-init . my-global-cns-boundary-mode) :init (add-to-list 'load-path my-cns-dir) (setq cns-prog (expand-file-name (if *is-windows* "cnws.exe" "cnws") my-cns-dir) cns-dict-directory (expand-file-name "cppjieba/dict" my-cns-dir) - cns-process-type 'shell) - (when *is-windows* - (setq cns-process-shell-command - (format "%s %s %s %s %s %s" - cns-prog - (concat (file-name-as-directory cns-dict-directory) "jieba.dict.utf8") - (concat (file-name-as-directory cns-dict-directory) "hmm_model.utf8") - (concat (file-name-as-directory cns-dict-directory) "user.dict.utf8") - (concat (file-name-as-directory cns-dict-directory) "idf.utf8") - (concat (file-name-as-directory cns-dict-directory) "stop_words.utf8")))) - - :config - (defun require-cns () - (require 'cns nil t)) - - (setq cns-recent-segmentation-limit 40) - (define-key cns-mode-map (kbd "C-<backspace>") nil)) + cns-recent-segmentation-limit 40 + cns-process-type 'shell + cns-process-shell-command + (format "%s %s %s %s %s %s" + cns-prog + (concat (file-name-as-directory cns-dict-directory) "jieba.dict.utf8") + (concat (file-name-as-directory cns-dict-directory) "hmm_model.utf8") + (concat (file-name-as-directory cns-dict-directory) "user.dict.utf8") + (concat (file-name-as-directory cns-dict-directory) "idf.utf8") + (concat (file-name-as-directory cns-dict-directory) "stop_words.utf8"))) + + ;; word segmentation via `find-word-boundary-function-table' + ;; NOT compatible with any other modes that use this char table + (defconst my-cns-boundary-empty-char-table (make-char-table nil) + "Assigned to `find-word-boundary-function-table' inside the +boundary function to avoid unwanted reentrancy.") + + (defconst my-cns-boundary-table + (let ((tab (make-char-table nil))) + (map-char-table + (lambda (key value) + (when (aref value ?C) + (set-char-table-range tab key #'my-cns-boundary-find-word-boundary))) + (standard-category-table)) + tab) + "Char table assigned to `find-word-boundary-function-table' in +`my-cns-boundary-mode' buffers.") + + (defun my-cns-boundary-han-run-at (pos) + "Return (START . END) of the maximal run of HànZì around POS." + (save-excursion + (goto-char pos) + (cons (if (re-search-backward "\\CC" nil t) (1+ (point)) (point-min)) + (progn (goto-char pos) + (if (re-search-forward "\\CC" nil t) (1- (point)) (point-max)))))) + + (defun my-cns-boundary-word-boundaries (pos) + "Return the ascending list of absolute word boundary positions of +the HànZì run around POS. e.g. (1 3 5) for \"中文分词\"." + (let* ((run (my-cns-boundary-han-run-at pos)) + (start (car run)) + (end (cdr run)) + (segs (cns-segmentation-handler + (buffer-substring-no-properties start end)))) + (let ((bounds (list start))) + (dolist (len segs) + (push (+ (car bounds) len) bounds)) + (setq bounds (nreverse bounds))))) + + (defun my-cns-boundary-find-word-boundary (pos limit) + "Find the Chinese word boundary near POS bounded by LIMIT.." + (let* ((find-word-boundary-function-table my-cns-boundary-empty-char-table) + (bounds (my-cns-boundary-word-boundaries pos))) + (if (< pos limit) + ;; POS is at the first character of a word. + (cl-find-if (lambda (b) (> b pos)) bounds) + ;; POS is at the last character of a word. + (car (last (cl-remove-if (lambda (b) (> b pos)) (butlast bounds))))))) + + (defun my-cns-boundary-mode-enable nil + "Enable `my-cns-boundary-mode'." + (unless (process-live-p cns-process) + (cns-start-process)) + (cns-buffer-list-add-buffer) + (setq-local find-word-boundary-function-table my-cns-boundary-table) + (add-hook 'kill-buffer-hook #'my-cns-boundary-mode-disable nil t)) + + (defun my-cns-boundary-mode-disable nil + "Disable `my-cns-boundary-mode'." + (cns-buffer-list-delete-buffer) + (kill-local-variable 'find-word-boundary-function-table) + (when (and (process-live-p cns-process) (= 0 (length cns-buffer-list))) + (cns-stop-process)) + (remove-hook 'kill-buffer-hook #'my-cns-boundary-mode-disable t)) + + (define-minor-mode my-cns-boundary-mode + "Toggle Chinese word segmentation mode in current buffer. +Unlike `cns-mode', this mode rebinds no keys." + :group 'cns + (if my-cns-boundary-mode (my-cns-boundary-mode-enable) (my-cns-boundary-mode-disable))) + + (define-globalized-minor-mode my-global-cns-boundary-mode my-cns-boundary-mode + (lambda nil (my-cns-boundary-mode 1)) + :group 'cns)) (provide 'init-segmentation) ;;; init-segmentation.el ends here