From ee7616d9563eabf2ae01927bc9d37ccf3e3b3325 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Wed, 18 Jan 2023 00:36:41 +0300 Subject: feat(corp/russian/data-import): new OpenCorpora data import tool Adds the beginning of a tool which can import OpenCorpora data into a SQLite database. This is quite a lot of toil and there's probably a better way to do this, but overall becoming this intimately familiar with the data structures is quite helpful for understanding what I can/can't do with only this dataset. Change-Id: Ieab33a8ce07ea4ac87917b9c8132226bbc6523b1 Reviewed-on: https://cl.tvl.fyi/c/depot/+/7859 Reviewed-by: tazjin Tested-by: BuildkiteCI --- corp/russian/data-import/default.nix | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 corp/russian/data-import/default.nix (limited to 'corp/russian/data-import/default.nix') diff --git a/corp/russian/data-import/default.nix b/corp/russian/data-import/default.nix new file mode 100644 index 000000000000..b4cdc50c25c1 --- /dev/null +++ b/corp/russian/data-import/default.nix @@ -0,0 +1,39 @@ +{ depot, pkgs, ... }: + +let + buildInputs = with pkgs; [ + sqlite + pkg-config + ]; + + # mirrored input data from OpenCorpora, as of 2023-01-17. + # + # This data is licensed under CC-BY-SA. + inputDataArchive = pkgs.fetchurl { + name = "dict.opcorpora.xml.bz"; + url = "https://tazj.in/blobs/dict.opcorpora.xml.bz2"; + sha256 = "04n5g43fkfc93z6xlwf2qfdrfdfl562pc2ivdb3cbbbsy56gkqg6"; + }; + + inputData = pkgs.runCommand "dict.opcorpora.xml" { } '' + ${pkgs.bzip2}/bin/bunzip2 -k -c ${inputDataArchive} > $out + ''; + + # development shell with native deps + shell = pkgs.mkShell { + inherit buildInputs; + + # make OPENCORPORA_DATA available in the environment + OPENCORPORA_DATA = inputData; + }; +in +depot.third_party.naersk.buildPackage { + src = depot.third_party.gitignoreSource ./.; + inherit buildInputs; + + passthru = { + inherit shell; + + + }; +} -- cgit 1.4.1