Gfllm search export

Dockerfile

@@ -145,7 +145,7 @@ ENV RIBOTIMEDIR="/usr/bin"
 ENV BIOEASELDIR="$RNA/Bio-Easel/blib/lib:$RNA/Bio-Easel/blib/arch:$RNA/Bio-Easel:$RNA/Bio-Easel/lib"
 ENV PERL5LIB="$BIOEASELDIR:$RIBODIR:$EPNOPTDIR:$EPNOFILEDIR:$EPNTESTDIR:$PERL5LIB"
 
-ENV PATH="$RNA/infernal-1.1.2/bin:$PATH"
+ENV PATH="$RNA/infernal-${INFERNAL_VERSION}/bin:$PATH"
 ENV PATH="$RNA/blat_suite:$PATH"
 ENV PATH="$RNA/seqkit:$PATH"
 ENV PATH="$RNACENTRAL_IMPORT_PIPELINE/bin:$PATH"

files/search-export/parts/goflow.sql

@@ -0,0 +1,13 @@
+COPY (
+  SELECT
+    json_build_object(
+      'id', todo.id,
+      'urs_taxid', todo.urs_taxid,
+      'should_show_goflow', true
+    )
+    FROM search_export_urs todo
+    JOIN go_flow_llm_curation_results gfllm
+    ON
+      todo.urs_taxid = gfllm.urs_taxid
+    ORDER by todo.id
+) TO STDOUT

rnacentral_pipeline/databases/ensembl/genomes/urls.py

@@ -30,8 +30,9 @@ def latest_release(ftp: FTP) -> str:
     readme_lines = []
     ftp.retrlines("RETR current_README", readme_lines.append)
     cur_readme = "\n".join(readme_lines)
-    pattern = r"Ensembl Release (\d+) Databases."
-    match = re.search(pattern, cur_readme)
+    pattern = r"Ensembl Release (\d+) Databases\."
+    match = re.search(pattern, cur_readme, re.IGNORECASE)
+
     if not match:
         raise ValueError("Could not find release number in README")
     release = match.group(1)

rnacentral_pipeline/databases/ensembl/vertebrates/urls.py

@@ -29,8 +29,9 @@ def latest_release(ftp: FTP) -> str:
     readme_lines = []
     ftp.retrlines("RETR current_README", readme_lines.append)
     cur_readme = "\n".join(readme_lines)
-    pattern = r"Ensembl Release (\d+) Databases."
-    match = re.search(pattern, cur_readme)
+    pattern = r"Ensembl Release (\d+) Databases\."
+    match = re.search(pattern, cur_readme, re.IGNORECASE)
+
     if not match:
         raise ValueError("Could not determine latest Ensembl release from README")
     release = match.group(1)

rnacentral_pipeline/rnacentral/r2dt/__init__.py

@@ -159,9 +159,7 @@ def prepare_sequences(xref_urs, tracked_urs, urs_to_fetch, max_sequences):
         .rename({"column_1": "urs"})
     )
 
-    raw_tracked = pl.scan_csv(
-        tracked_urs.name, low_memory=True
-    ).unique() 
+    raw_tracked = pl.scan_csv(tracked_urs.name, low_memory=True).unique()
 
     to_fetch = raw_xref.join(raw_tracked, on="urs", how="anti")
 

rnacentral_pipeline/rnacentral/r2dt/data.py

@@ -409,11 +409,15 @@ def dot_bracket(self):
             seq_dot = str(record.seq)
             ## Use indices instead, assert that the string is even length
             ## If not, then the two parts are not the same length
-            assert len(seq_dot) % 2 == 0, f"Odd length sequence {len(seq_dot)}"
+            if len(seq_dot) % 2 != 0:
+                raise ValueError(f"Odd length sequence {len(seq_dot)}")
             seq_dot_len = len(seq_dot)
             sequence = seq_dot[0 : seq_dot_len // 2]
             dot_bracket = seq_dot[(seq_dot_len // 2) :]
-            assert len(sequence) == len(dot_bracket)
+            if len(sequence) != len(dot_bracket):
+                raise ValueError(
+                    f"Sequence and dot bracket lengths do not match: {len(sequence)} != {len(dot_bracket)}"
+                )
             return dot_bracket
 
     def basepair_count(self):

rnacentral_pipeline/rnacentral/search_export/data.py

@@ -709,6 +709,10 @@ def has_litsumm(litsumm):
     return str(bool(litsumm))
 
 
+def has_go_flow_llm_annotation(go_flow):
+    return str(bool(go_flow))
+
+
 def has_editing_event(editing_events):
     return str(bool(editing_events))
 
@@ -880,6 +884,11 @@ def edit_ref_to_edit(editing_events):
                     edit_repeat_type,
                     keys="editing_events",
                 ),
+                field(
+                    "has_go_flow_llm_annotation",
+                    has_go_flow_llm_annotation,
+                    keys="go_flow_llm_annotations",
+                ),
                 ## Add new fields above this line! Otherwise editing the produced xml is hard.
                 tree("so_rna_type", so_rna_type_tree, key="so_rna_type_tree"),
             ],

utils/precompute/Cargo.toml

@@ -20,7 +20,6 @@ sorted-iter = "0.1.7"
 structopt = "0.3"
 strum = "0.21"
 strum_macros = "0.21"
-polars = { version = "0.33.2", features = ["lazy", "streaming"] }
 
 [dev-dependencies]
 rand = "0.8"

utils/precompute/src/releases.rs

@@ -26,7 +26,6 @@ use anyhow::{
     Result,
 };
 
-use polars::prelude::*;
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct UrsEntry {

utils/search-export/src/main.rs

@@ -34,6 +34,7 @@ pub enum Groupable {
     SoInfo,
     LitsummSummaries,
     EditingEvents,
+    GoFlowAnnotation,
 }
 
 #[derive(Debug, StructOpt)]
@@ -145,6 +146,10 @@ enum SequenceCommand {
         /// RNA editing events
         editing_events: PathBuf,
 
+        #[structopt(parse(from_os_str))]
+        /// GoFlowLLM annotations
+        go_flow_llm_annotations: PathBuf,
+
         // Add new arguments above this line!
         #[structopt(parse(from_os_str))]
         /// Filename to write the results to, '-' means stdout
@@ -260,6 +265,9 @@ fn main() -> Result<()> {
             Groupable::EditingEvents => {
                 sequences::editing_events::group(&path, max_count, &output)?
             },
+            Groupable::GoFlowAnnotation => {
+                sequences::go_flow_annotations::group(&path, max_count, &output)?
+            },
         },
         Subcommand::Sequences {
             command,
@@ -280,6 +288,8 @@ fn main() -> Result<()> {
                 litsumm_summaries,
                 editing_events,
                 so_term_tree,
+                go_flow_llm_annotations,
+                // Add new arguments above this line!
                 output,
             } => sequences::writers::write_merge(
                 vec![
@@ -298,6 +308,7 @@ fn main() -> Result<()> {
                     editing_events,
                     orfs,
                     so_term_tree,
+                    go_flow_llm_annotations,
                 ],
                 &output,
             )?,

utils/search-export/src/sequences/file_joiner.rs

@@ -37,6 +37,7 @@ use super::{
     editing_events::EditingEvent,
     feedback::Feedback,
     go_annotation::GoAnnotation,
+    go_flow_annotations::GoFlowLLMAnnotation,
     interacting_protein::InteractingProtein,
     interacting_rna::InteractingRna,
     litsumm::LitsummSummaries,
@@ -98,6 +99,7 @@ pub enum FileTypes {
     PublicationCount,
     LitsummSummaries,
     EditingEvents,
+    GoFlowLLMAnnotations,
     SoTermTree,
 }
 
@@ -116,6 +118,8 @@ pub struct FileJoiner<'de> {
     rfam_hits: StreamDeserializer<'de, IoRead<BufReader<File>>, Grouped<RfamHit>>,
     publication_counts: StreamDeserializer<'de, IoRead<BufReader<File>>, Grouped<PublicationCount>>,
     lit_summ: StreamDeserializer<'de, IoRead<BufReader<File>>, Grouped<LitsummSummaries>>,
+    go_flow_llm_annotations:
+        StreamDeserializer<'de, IoRead<BufReader<File>>, Grouped<GoFlowLLMAnnotation>>,
     editing_events: StreamDeserializer<'de, IoRead<BufReader<File>>, Grouped<EditingEvent>>,
     so_info: SoMapping,
 }
@@ -203,6 +207,7 @@ impl FileJoinerBuilder {
         let publication_counts = self.iterator_for(FileTypes::PublicationCount)?;
         let lit_summ = self.iterator_for(FileTypes::LitsummSummaries)?;
         let editing_events = self.iterator_for(FileTypes::EditingEvents)?;
+        let go_flow_llm_annotations = self.iterator_for(FileTypes::GoFlowLLMAnnotations)?;
         let so_info = so_tree::load(self.path_for(FileTypes::SoTermTree)?)?;
 
         Ok(FileJoiner {
@@ -220,6 +225,7 @@ impl FileJoinerBuilder {
             publication_counts,
             lit_summ,
             editing_events,
+            go_flow_llm_annotations,
             so_info,
         })
     }
@@ -244,6 +250,7 @@ impl<'de> Iterator for FileJoiner<'de> {
             self.publication_counts.next(),
             self.lit_summ.next(),
             self.editing_events.next(),
+            self.go_flow_llm_annotations.next(),
         );
 
         match current {
@@ -262,6 +269,7 @@ impl<'de> Iterator for FileJoiner<'de> {
                 None,
                 None,
                 None,
+                None,
             ) => None,
             (
                 Some(Ok(Required {
@@ -320,6 +328,10 @@ impl<'de> Iterator for FileJoiner<'de> {
                     id: id14,
                     data: editing_events,
                 })),
+                Some(Ok(Multiple {
+                    id: id15,
+                    data: goflow_llm_annotations,
+                })),
             ) => {
                 if id1 != id2
                     || id1 != id3
@@ -334,9 +346,11 @@ impl<'de> Iterator for FileJoiner<'de> {
                     || id1 != id12
                     || id1 != id13
                     || id1 != id14
+                    || id1 != id15
                 {
                     return Some(Err(Error::OutofSyncData(vec![
                         id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, id11, id12, id13, id14,
+                        id15,
                     ])));
                 }
 
@@ -362,6 +376,7 @@ impl<'de> Iterator for FileJoiner<'de> {
                     .publication_counts(publication_counts)
                     .litsumm_summaries(lit_summ)
                     .editing_events(editing_events)
+                    .go_flow_llm_annotations(goflow_llm_annotations)
                     .so_tree(so_tree)
                     .build();
 

utils/search-export/src/sequences/go_flow_annotations.rs

@@ -0,0 +1,35 @@
+use serde::{
+    Deserialize,
+    Serialize,
+};
+use std::path::Path;
+
+use anyhow::Result;
+use rnc_core::grouper;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct GoFlowLLMAnnotation {
+    pub id: usize,
+    urs_taxid: String,
+    should_show_goflow: bool,
+}
+
+impl grouper::HasIndex for GoFlowLLMAnnotation {
+    fn index(&self) -> usize {
+        self.id
+    }
+}
+
+pub fn group(path: &Path, max: usize, output: &Path) -> Result<()> {
+    grouper::group::<GoFlowLLMAnnotation>(grouper::Criteria::AnyNumber, &path, 1, max, &output)
+}
+
+impl GoFlowLLMAnnotation {
+    pub fn should_show_goflow(&self) -> bool {
+        self.should_show_goflow
+    }
+
+    pub fn urs_taxid(&self) -> &str {
+        &self.urs_taxid
+    }
+}

utils/search-export/src/sequences/mod.rs

@@ -5,6 +5,7 @@ pub mod editing_events;
 pub mod feedback;
 pub mod file_joiner;
 pub mod go_annotation;
+pub mod go_flow_annotations;
 pub mod interacting_protein;
 pub mod interacting_rna;
 pub mod litsumm;

utils/search-export/src/sequences/normalized.rs

@@ -38,6 +38,7 @@ use crate::{
         editing_events::EditingEvent,
         feedback::FeedbackVec,
         go_annotation::GoAnnotation,
+        go_flow_annotations::GoFlowLLMAnnotation,
         interacting_protein::InteractingProtein,
         interacting_rna::InteractingRna,
         litsumm::LitsummSummaries,
@@ -80,6 +81,7 @@ pub struct Normalized {
     publication_count: usize,
     litsumm: Vec<LitsummSummaries>,
     editing_events: Vec<EditingEvent>,
+    go_flow_llm_annotations: Vec<GoFlowLLMAnnotation>,
     so_rna_type_tree: so_tree::SoTree,
 
     #[serde(flatten)]
@@ -140,6 +142,7 @@ impl Normalized {
             rfam_hits: raw.rfam_hits().iter().cloned().collect(),
             orfs: raw.orfs().iter().cloned().collect(),
             litsumm: raw.litsumm_summaries().to_vec(),
+            go_flow_llm_annotations: raw.go_flow_llm_annotations().to_vec(),
             editing_events: raw.editing_events().to_vec(),
         })
     }

utils/search-export/src/sequences/raw.rs

@@ -16,6 +16,7 @@ use crate::sequences::{
     editing_events::EditingEvent,
     feedback::Feedback,
     go_annotation::GoAnnotation,
+    go_flow_annotations::GoFlowLLMAnnotation,
     interacting_protein::InteractingProtein,
     interacting_rna::InteractingRna,
     litsumm::LitsummSummaries,
@@ -46,6 +47,7 @@ pub struct Raw {
     publication_counts: Option<PublicationCount>,
     litsumm_summaries: Vec<LitsummSummaries>,
     editing_events: Vec<EditingEvent>,
+    go_flow_llm_annotations: Vec<GoFlowLLMAnnotation>,
     so_tree: so_tree::SoTree,
 }
 
@@ -148,6 +150,11 @@ impl Raw {
         &self.editing_events
     }
 
+    /// Get a reference to the raw's GoFlowlLM annotations
+    pub fn go_flow_llm_annotations(&self) -> &[GoFlowLLMAnnotation] {
+        &self.go_flow_llm_annotations
+    }
+
     /// Get this raw's publication count.
     pub fn publication_count(&self) -> usize {
         self.publication_counts.as_ref().map(|p| p.publication_count()).unwrap_or(0)