diff options
-rw-r--r-- | corp/russian/data-import/src/db_setup.rs | 39 | ||||
-rw-r--r-- | corp/russian/data-import/src/main.rs | 21 | ||||
-rw-r--r-- | corp/russian/data-import/src/or_parser.rs | 15 |
3 files changed, 69 insertions, 6 deletions
diff --git a/corp/russian/data-import/src/db_setup.rs b/corp/russian/data-import/src/db_setup.rs index 5fe64717ad9b..4644edf09424 100644 --- a/corp/russian/data-import/src/db_setup.rs +++ b/corp/russian/data-import/src/db_setup.rs @@ -6,7 +6,7 @@ //! introduce things like foreign key constraints between tables that //! represent relations. -use super::{bail, Ensure}; +use super::Ensure; use crate::oc_parser::*; use crate::or_parser; use log::{debug, info}; @@ -181,6 +181,16 @@ CREATE TABLE or_words ( word_type TEXT, level TEXT ) STRICT; + +CREATE TABLE or_words_forms ( + id INTEGER PRIMARY KEY, + word_id INTEGER NOT NULL, + form_type TEXT, + position TEXT, + form TEXT, + form_bare TEXT, + FOREIGN KEY(word_id) REFERENCES words(id) +) STRICT; "#, ) .ensure("setting up OpenRussian table schema failed"); @@ -215,3 +225,30 @@ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) info!("inserted {} OpenRussian words", count); } + +pub fn insert_or_word_forms<I: Iterator<Item = or_parser::WordForm>>(conn: &Connection, forms: I) { + let mut stmt = conn + .prepare_cached( + " +INSERT INTO or_words_forms (id, word_id, form_type, position, form, form_bare) +VALUES (?1, ?2, ?3, ?4, ?5, ?6) +", + ) + .ensure("failed to prepare OR word forms statement"); + let mut count = 0; + + for form in forms { + stmt.execute(( + form.id, + form.word_id, + form.form_type, + form.position, + form.form, + form.form_bare, + )) + .ensure("failed to insert OR word form"); + count += 1; + } + + info!("inserted {} OpenRussian word forms", count); +} diff --git a/corp/russian/data-import/src/main.rs b/corp/russian/data-import/src/main.rs index 11387539ab84..18bc0238e7e0 100644 --- a/corp/russian/data-import/src/main.rs +++ b/corp/russian/data-import/src/main.rs @@ -210,12 +210,23 @@ fn open_russian(conn: &Connection, args: &Args) { db_setup::initial_or_schema(conn); - let tx = conn - .unchecked_transaction() - .ensure("failed to start transaction"); + { + let tx = conn + .unchecked_transaction() + .ensure("failed to start transaction"); + + db_setup::insert_or_words(&tx, parser.words()); + tx.commit().ensure("OpenRussian words commit failed"); + } + + { + let tx = conn + .unchecked_transaction() + .ensure("failed to start transaction"); - db_setup::insert_or_words(&tx, parser.words()); - tx.commit().ensure("OpenRussian words commit failed"); + db_setup::insert_or_word_forms(&tx, parser.words_forms()); + tx.commit().ensure("OpenRussian word forms commit failed"); + } info!("finished OpenRussian import"); } diff --git a/corp/russian/data-import/src/or_parser.rs b/corp/russian/data-import/src/or_parser.rs index c11896f6bac0..28e4f14d3169 100644 --- a/corp/russian/data-import/src/or_parser.rs +++ b/corp/russian/data-import/src/or_parser.rs @@ -33,6 +33,17 @@ pub struct Word { pub created_at: String, // TODO: unknown } +/// A word form from the `words_forms` table. +#[derive(Debug, Deserialize)] +pub struct WordForm { + pub id: usize, + pub word_id: usize, + pub form_type: String, + pub position: String, + pub form: String, + pub form_bare: String, +} + pub struct OpenRussianParser { or_directory: PathBuf, } @@ -50,6 +61,10 @@ impl OpenRussianParser { self.parser_for("words.csv") } + pub fn words_forms(&self) -> DynIter<WordForm> { + self.parser_for("words_forms.csv") + } + fn parser_for<T: serde::de::DeserializeOwned + 'static>( &self, file_name: &str, |