From 429c0d00c4cd07ea90c85bf1ec2f2c742d970420 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Fri, 20 Jan 2023 13:31:12 +0300 Subject: feat(corp/data-import): add import of OpenRussian 'words' table This is actually the lemmata table of this corpus, not the forms of all words (they're in a separate table). Change-Id: I89a2c2817ccce840f47406fa2a636f4ed3f49154 Reviewed-on: https://cl.tvl.fyi/c/depot/+/7893 Reviewed-by: tazjin Tested-by: BuildkiteCI --- corp/russian/data-import/default.nix | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'corp/russian/data-import/default.nix') diff --git a/corp/russian/data-import/default.nix b/corp/russian/data-import/default.nix index cf358874dce6..6aa8ad6aa3d8 100644 --- a/corp/russian/data-import/default.nix +++ b/corp/russian/data-import/default.nix @@ -19,6 +19,9 @@ let ${pkgs.bzip2}/bin/bunzip2 -k -c ${openCorporaArchive} > $out ''; + # mirrored input data from OpenRussian, as of 2023-01-17. + # + # This data is licensed under CC-BY-SA. openRussianArchive = pkgs.fetchzip { name = "openrussian-20230117"; url = "https://tazj.in/blobs/openrussian-20230117.tar.xz"; @@ -43,8 +46,10 @@ lib.fix (self: depot.third_party.naersk.buildPackage { inherit shell openCorpora; # target that actually builds an entire database - database = pkgs.runCommand "tvl-russian-db.sqlite" { } '' - ${self}/bin/data-import ${openCorpora} $out - ''; + database = pkgs.runCommand "tvl-russian-db.sqlite" + { + OPENCORPORA_DATA = openCorpora; + OPENRUSSIAN_DATA = openRussianArchive; + } "${self}/bin/data-import --output $out"; }; }) -- cgit 1.4.1