From 1408aa301a61dba37a1a60caa83396c1193571a8 Mon Sep 17 00:00:00 2001 From: ahmedlone127 Date: Fri, 7 Nov 2025 11:08:35 +0500 Subject: [PATCH 1/8] introducing perfered engine logic --- .../annotator/audio/hubert_for_ctc.py | 4 +- .../annotator/audio/wav2vec2_for_ctc.py | 4 +- .../annotator/audio/whisper_for_ctc.py | 4 +- .../albert_for_multiple_choice.py | 4 +- .../albert_for_question_answering.py | 4 +- .../albert_for_sequence_classification.py | 4 +- .../albert_for_token_classification.py | 4 +- .../albert_for_zero_shot_classification.py | 4 +- .../bart_for_zero_shot_classification.py | 4 +- .../classifier_dl/bert_for_multiple_choice.py | 4 +- .../bert_for_question_answering.py | 4 +- .../bert_for_sequence_classification.py | 4 +- .../bert_for_token_classification.py | 4 +- .../bert_for_zero_shot_classification.py | 4 +- .../camembert_for_question_answering.py | 4 +- .../camembert_for_sequence_classification.py | 4 +- .../camembert_for_token_classification.py | 4 +- .../camembert_for_zero_shot_classification.py | 4 +- .../annotator/classifier_dl/classifier_dl.py | 4 +- .../deberta_for_question_answering.py | 4 +- .../deberta_for_sequence_classification.py | 4 +- .../deberta_for_token_classification.py | 4 +- .../deberta_for_zero_shot_classification.py | 4 +- .../distil_bert_for_question_answering.py | 4 +- ...distil_bert_for_sequence_classification.py | 4 +- .../distil_bert_for_token_classification.py | 4 +- ...istil_bert_for_zero_shot_classification.py | 4 +- .../distilbert_for_multiple_choice.py | 4 +- .../longformer_for_question_answering.py | 4 +- .../longformer_for_sequence_classification.py | 4 +- .../longformer_for_token_classification.py | 4 +- .../mpnet_for_question_answering.py | 4 +- .../mpnet_for_sequence_classification.py | 4 +- .../mpnet_for_token_classification.py | 4 +- .../classifier_dl/multi_classifier_dl.py | 4 +- .../roberta_for_multiple_choice.py | 4 +- .../roberta_for_question_answering.py | 4 +- .../roberta_for_sequence_classification.py | 4 +- .../roberta_for_token_classification.py | 4 +- .../roberta_for_zero_shot_classification.py | 4 +- .../annotator/classifier_dl/sentiment_dl.py | 4 +- .../tapas_for_question_answering.py | 4 +- .../xlm_roberta_for_multiple_choice.py | 4 +- .../xlm_roberta_for_question_answering.py | 4 +- ...xlm_roberta_for_sequence_classification.py | 6 +-- .../xlm_roberta_for_token_classification.py | 4 +- ...lm_roberta_for_zero_shot_classification.py | 4 +- .../xlnet_for_sequence_classification.py | 4 +- .../xlnet_for_token_classification.py | 4 +- .../annotator/coref/spanbert_coref.py | 4 +- .../cv/blip_for_question_answering.py | 4 +- .../cv/clip_for_zero_shot_classification.py | 4 +- .../cv/convnext_for_image_classification.py | 4 +- .../annotator/cv/florence2_transformer.py | 4 +- .../annotator/cv/gemma3_for_multimodal.py | 4 +- .../annotator/cv/internvl_for_multimodal.py | 4 +- .../annotator/cv/janus_for_multimodal.py | 4 +- .../annotator/cv/llava_for_multimodal.py | 4 +- .../annotator/cv/mllama_for_multimodal.py | 4 +- .../annotator/cv/paligemma_for_multimodal.py | 4 +- .../cv/phi3_vision_for_multimodal.py | 4 +- .../annotator/cv/qwen2vl_transformer.py | 4 +- .../annotator/cv/smolvlm_transformer.py | 4 +- .../cv/swin_for_image_classification.py | 4 +- ...on_encoder_decoder_for_image_captioning.py | 4 +- .../cv/vit_for_image_classification.py | 4 +- .../annotator/dependency/dependency_parser.py | 4 +- .../dependency/typed_dependency_parser.py | 4 +- .../annotator/embeddings/albert_embeddings.py | 4 +- .../embeddings/auto_gguf_embeddings.py | 4 +- .../annotator/embeddings/bert_embeddings.py | 4 +- .../embeddings/bert_sentence_embeddings.py | 4 +- .../annotator/embeddings/bge_embeddings.py | 4 +- .../embeddings/camembert_embeddings.py | 4 +- .../embeddings/deberta_embeddings.py | 4 +- .../embeddings/distil_bert_embeddings.py | 4 +- .../sparknlp/annotator/embeddings/doc2vec.py | 4 +- .../annotator/embeddings/e5_embeddings.py | 4 +- .../annotator/embeddings/e5v_embeddings.py | 4 +- .../annotator/embeddings/elmo_embeddings.py | 4 +- .../embeddings/instructor_embeddings.py | 4 +- .../embeddings/longformer_embeddings.py | 4 +- .../annotator/embeddings/minilm_embeddings.py | 4 +- .../annotator/embeddings/mpnet_embeddings.py | 4 +- .../annotator/embeddings/mxbai_embeddings.py | 4 +- .../annotator/embeddings/nomic_embeddings.py | 4 +- .../embeddings/roberta_embeddings.py | 4 +- .../embeddings/roberta_sentence_embeddings.py | 4 +- .../embeddings/snowflake_embeddings.py | 4 +- .../annotator/embeddings/uae_embeddings.py | 4 +- .../embeddings/universal_sentence_encoder.py | 4 +- .../sparknlp/annotator/embeddings/word2vec.py | 4 +- .../annotator/embeddings/word_embeddings.py | 4 +- .../embeddings/xlm_roberta_embeddings.py | 4 +- .../xlm_roberta_sentence_embeddings.py | 4 +- .../annotator/embeddings/xlnet_embeddings.py | 4 +- .../annotator/ld_dl/language_detector_dl.py | 4 +- python/sparknlp/annotator/lemmatizer.py | 4 +- .../annotator/matcher/big_text_matcher.py | 4 +- .../annotator/matcher/text_matcher.py | 4 +- python/sparknlp/annotator/ner/ner_crf.py | 4 +- python/sparknlp/annotator/ner/ner_dl.py | 4 +- .../annotator/ner/zero_shot_ner_model.py | 4 +- python/sparknlp/annotator/pos/perceptron.py | 4 +- .../sentence/sentence_detector_dl.py | 4 +- .../annotator/sentiment/vivekn_sentiment.py | 4 +- .../annotator/seq2seq/auto_gguf_model.py | 4 +- .../annotator/seq2seq/auto_gguf_reranker.py | 4 +- .../seq2seq/auto_gguf_vision_model.py | 4 +- .../annotator/seq2seq/bart_transformer.py | 4 +- .../annotator/seq2seq/cohere_transformer.py | 4 +- .../annotator/seq2seq/cpm_transformer.py | 4 +- .../annotator/seq2seq/gpt2_transformer.py | 4 +- .../annotator/seq2seq/llama2_transformer.py | 4 +- .../annotator/seq2seq/llama3_transformer.py | 4 +- .../annotator/seq2seq/m2m100_transformer.py | 4 +- .../annotator/seq2seq/marian_transformer.py | 4 +- .../annotator/seq2seq/mistral_transformer.py | 4 +- .../annotator/seq2seq/nllb_transformer.py | 4 +- .../annotator/seq2seq/olmo_transformer.py | 4 +- .../annotator/seq2seq/phi2_transformer.py | 4 +- .../annotator/seq2seq/phi3_transformer.py | 4 +- .../annotator/seq2seq/phi4_transformer.py | 10 ++-- .../annotator/seq2seq/qwen_transformer.py | 4 +- .../seq2seq/starcoder_transformer.py | 4 +- .../annotator/seq2seq/t5_transformer.py | 4 +- .../spell_check/context_spell_checker.py | 4 +- .../annotator/spell_check/norvig_sweeting.py | 4 +- .../annotator/spell_check/symmetric_delete.py | 4 +- .../sparknlp/annotator/stop_words_cleaner.py | 4 +- python/sparknlp/annotator/token/tokenizer.py | 4 +- .../sparknlp/annotator/ws/word_segmenter.py | 4 +- python/sparknlp/internal/__init__.py | 7 ++- .../pretrained/resource_downloader.py | 6 +-- ...istilbert_for_token_classification_test.py | 8 ++-- .../com/johnsnowlabs/nlp/HasPretrained.scala | 30 ++++++++++-- .../annotators/seq2seq/Phi4Transformer.scala | 3 +- .../nlp/pretrained/ResourceDownloader.scala | 43 +++++++++++++---- .../nlp/pretrained/ResourceMetadata.scala | 47 +++++++++++++++---- 139 files changed, 378 insertions(+), 302 deletions(-) diff --git a/python/sparknlp/annotator/audio/hubert_for_ctc.py b/python/sparknlp/annotator/audio/hubert_for_ctc.py index 0f7549e712ae4e..65e4c03d5ec2ac 100644 --- a/python/sparknlp/annotator/audio/hubert_for_ctc.py +++ b/python/sparknlp/annotator/audio/hubert_for_ctc.py @@ -165,7 +165,7 @@ def loadSavedModel(folder, spark_session): return HubertForCTC(java_model=jModel) @staticmethod - def pretrained(name="asr_hubert_large_ls960", lang="en", remote_loc=None): + def pretrained(name="asr_hubert_large_ls960", lang="en", remote_loc=None, engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -185,4 +185,4 @@ def pretrained(name="asr_hubert_large_ls960", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(HubertForCTC, name, lang, remote_loc) + return ResourceDownloader.downloadModel(HubertForCTC, name, lang, remote_loc, engine) diff --git a/python/sparknlp/annotator/audio/wav2vec2_for_ctc.py b/python/sparknlp/annotator/audio/wav2vec2_for_ctc.py index c1937df094e5c8..f4f3fe9f23bd94 100644 --- a/python/sparknlp/annotator/audio/wav2vec2_for_ctc.py +++ b/python/sparknlp/annotator/audio/wav2vec2_for_ctc.py @@ -138,7 +138,7 @@ def loadSavedModel(folder, spark_session): return Wav2Vec2ForCTC(java_model=jModel) @staticmethod - def pretrained(name="asr_wav2vec2_base_960h", lang="en", remote_loc=None): + def pretrained(name="asr_wav2vec2_base_960h", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -158,4 +158,4 @@ def pretrained(name="asr_wav2vec2_base_960h", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Wav2Vec2ForCTC, name, lang, remote_loc) + return ResourceDownloader.downloadModel(Wav2Vec2ForCTC, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/audio/whisper_for_ctc.py b/python/sparknlp/annotator/audio/whisper_for_ctc.py index 343bf4749a3714..446d3116ae138a 100644 --- a/python/sparknlp/annotator/audio/whisper_for_ctc.py +++ b/python/sparknlp/annotator/audio/whisper_for_ctc.py @@ -228,7 +228,7 @@ def loadSavedModel(folder, spark_session): return WhisperForCTC(java_model=jModel) @staticmethod - def pretrained(name="asr_whisper_tiny_opt", lang="xx", remote_loc=None): + def pretrained(name="asr_whisper_tiny_opt", lang="xx", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -248,4 +248,4 @@ def pretrained(name="asr_whisper_tiny_opt", lang="xx", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(WhisperForCTC, name, lang, remote_loc) + return ResourceDownloader.downloadModel(WhisperForCTC, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/albert_for_multiple_choice.py b/python/sparknlp/annotator/classifier_dl/albert_for_multiple_choice.py index 7dc610b256f687..25d5b6391c57fd 100644 --- a/python/sparknlp/annotator/classifier_dl/albert_for_multiple_choice.py +++ b/python/sparknlp/annotator/classifier_dl/albert_for_multiple_choice.py @@ -138,7 +138,7 @@ def loadSavedModel(folder, spark_session): return AlbertForMultipleChoice(java_model=jModel) @staticmethod - def pretrained(name="albert_base_uncased_multiple_choice", lang="en", remote_loc=None): + def pretrained(name="albert_base_uncased_multiple_choice", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -158,4 +158,4 @@ def pretrained(name="albert_base_uncased_multiple_choice", lang="en", remote_loc The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertForMultipleChoice, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(AlbertForMultipleChoice, name, lang, remote_loc,engine ) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/albert_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/albert_for_question_answering.py index 113cdbfb801184..b771f754e26d11 100755 --- a/python/sparknlp/annotator/classifier_dl/albert_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/albert_for_question_answering.py @@ -149,7 +149,7 @@ def loadSavedModel(folder, spark_session): return AlbertForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="albert_base_qa_squad2", lang="en", remote_loc=None): + def pretrained(name="albert_base_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -169,4 +169,4 @@ def pretrained(name="albert_base_qa_squad2", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertForQuestionAnswering, name, lang, remote_loc) + return ResourceDownloader.downloadModel(AlbertForQuestionAnswering, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/albert_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/albert_for_sequence_classification.py index fa9e7b5168b709..c7aa03f15cc0eb 100755 --- a/python/sparknlp/annotator/classifier_dl/albert_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/albert_for_sequence_classification.py @@ -178,7 +178,7 @@ def loadSavedModel(folder, spark_session): return AlbertForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="albert_base_sequence_classifier_imdb", lang="en", remote_loc=None): + def pretrained(name="albert_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -198,4 +198,4 @@ def pretrained(name="albert_base_sequence_classifier_imdb", lang="en", remote_lo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertForSequenceClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(AlbertForSequenceClassification, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/albert_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/albert_for_token_classification.py index 814ec342a5589b..8c8e569c10ba15 100755 --- a/python/sparknlp/annotator/classifier_dl/albert_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/albert_for_token_classification.py @@ -156,7 +156,7 @@ def loadSavedModel(folder, spark_session): return AlbertForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="albert_base_token_classifier_conll03", lang="en", remote_loc=None): + def pretrained(name="albert_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -176,4 +176,4 @@ def pretrained(name="albert_base_token_classifier_conll03", lang="en", remote_lo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertForTokenClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(AlbertForTokenClassification, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/albert_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/albert_for_zero_shot_classification.py index 4ea47dab83166e..9ffb4a1c5cc27c 100644 --- a/python/sparknlp/annotator/classifier_dl/albert_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/albert_for_zero_shot_classification.py @@ -188,7 +188,7 @@ def loadSavedModel(folder, spark_session): return AlbertForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="albert_zero_shot_classifier_onnx", lang="en", remote_loc=None): + def pretrained(name="albert_zero_shot_classifier_onnx", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -208,4 +208,4 @@ def pretrained(name="albert_zero_shot_classifier_onnx", lang="en", remote_loc=No The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertForZeroShotClassification, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(AlbertForZeroShotClassification, name, lang, remote_loc,engine ) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/bart_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/bart_for_zero_shot_classification.py index 8a05f20805b2d8..da45cb89289b00 100644 --- a/python/sparknlp/annotator/classifier_dl/bart_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/bart_for_zero_shot_classification.py @@ -202,7 +202,7 @@ def loadSavedModel(folder, spark_session): return BartForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="bart_large_zero_shot_classifier_mnli", lang="en", remote_loc=None): + def pretrained(name="bart_large_zero_shot_classifier_mnli", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -222,4 +222,4 @@ def pretrained(name="bart_large_zero_shot_classifier_mnli", lang="en", remote_lo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BartForZeroShotClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(BartForZeroShotClassification, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/bert_for_multiple_choice.py b/python/sparknlp/annotator/classifier_dl/bert_for_multiple_choice.py index 045e8d64180b53..2854d0b3f46c2c 100644 --- a/python/sparknlp/annotator/classifier_dl/bert_for_multiple_choice.py +++ b/python/sparknlp/annotator/classifier_dl/bert_for_multiple_choice.py @@ -138,7 +138,7 @@ def loadSavedModel(folder, spark_session): return BertForMultipleChoice(java_model=jModel) @staticmethod - def pretrained(name="bert_base_uncased_multiple_choice", lang="en", remote_loc=None): + def pretrained(name="bert_base_uncased_multiple_choice", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -158,4 +158,4 @@ def pretrained(name="bert_base_uncased_multiple_choice", lang="en", remote_loc=N The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertForMultipleChoice, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(BertForMultipleChoice, name, lang, remote_loc,engine ) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/bert_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/bert_for_question_answering.py index 27b61bdccdd953..fa1f07b5750b4d 100755 --- a/python/sparknlp/annotator/classifier_dl/bert_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/bert_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return BertForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="bert_base_cased_qa_squad2", lang="en", remote_loc=None): + def pretrained(name="bert_base_cased_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="bert_base_cased_qa_squad2", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertForQuestionAnswering, name, lang, remote_loc) + return ResourceDownloader.downloadModel(BertForQuestionAnswering, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/bert_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/bert_for_sequence_classification.py index 8ae3dbf185df6a..fae880b5bf5fa2 100755 --- a/python/sparknlp/annotator/classifier_dl/bert_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/bert_for_sequence_classification.py @@ -179,7 +179,7 @@ def loadSavedModel(folder, spark_session): return BertForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="bert_base_sequence_classifier_imdb", lang="en", remote_loc=None): + def pretrained(name="bert_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -199,4 +199,4 @@ def pretrained(name="bert_base_sequence_classifier_imdb", lang="en", remote_loc= The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertForSequenceClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(BertForSequenceClassification, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/bert_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/bert_for_token_classification.py index bef1f945b7ba06..5b9ff56b6fdfbe 100755 --- a/python/sparknlp/annotator/classifier_dl/bert_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/bert_for_token_classification.py @@ -154,7 +154,7 @@ def loadSavedModel(folder, spark_session): return BertForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="bert_base_token_classifier_conll03", lang="en", remote_loc=None): + def pretrained(name="bert_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -174,4 +174,4 @@ def pretrained(name="bert_base_token_classifier_conll03", lang="en", remote_loc= The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertForTokenClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(BertForTokenClassification, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/bert_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/bert_for_zero_shot_classification.py index e2058c6c77f8c1..900faea1b084cd 100755 --- a/python/sparknlp/annotator/classifier_dl/bert_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/bert_for_zero_shot_classification.py @@ -189,7 +189,7 @@ def loadSavedModel(folder, spark_session): return BertForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="bert_zero_shot_classifier_mnli", lang="xx", remote_loc=None): + def pretrained(name="bert_zero_shot_classifier_mnli", lang="xx", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -209,4 +209,4 @@ def pretrained(name="bert_zero_shot_classifier_mnli", lang="xx", remote_loc=None The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertForZeroShotClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(BertForZeroShotClassification, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/camembert_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/camembert_for_question_answering.py index 02828e034c5125..938f4586899563 100755 --- a/python/sparknlp/annotator/classifier_dl/camembert_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/camembert_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return CamemBertForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="camembert_base_qa_fquad", lang="fr", remote_loc=None): + def pretrained(name="camembert_base_qa_fquad", lang="fr", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="camembert_base_qa_fquad", lang="fr", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CamemBertForQuestionAnswering, name, lang, remote_loc) + return ResourceDownloader.downloadModel(CamemBertForQuestionAnswering, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/camembert_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/camembert_for_sequence_classification.py index 305f9d9453baf0..28e1c36d50bdc8 100644 --- a/python/sparknlp/annotator/classifier_dl/camembert_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/camembert_for_sequence_classification.py @@ -182,7 +182,7 @@ def loadSavedModel(folder, spark_session): return CamemBertForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="camembert_base_sequence_classifier_allocine", lang="fr", remote_loc=None): + def pretrained(name="camembert_base_sequence_classifier_allocine", lang="fr", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -202,4 +202,4 @@ def pretrained(name="camembert_base_sequence_classifier_allocine", lang="fr", re The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CamemBertForSequenceClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(CamemBertForSequenceClassification, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/camembert_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/camembert_for_token_classification.py index 19fd4780f8e583..94f466d26d7be5 100755 --- a/python/sparknlp/annotator/classifier_dl/camembert_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/camembert_for_token_classification.py @@ -150,7 +150,7 @@ def loadSavedModel(folder, spark_session): return CamemBertForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="camembert_base_token_classifier_wikiner", lang="en", remote_loc=None): + def pretrained(name="camembert_base_token_classifier_wikiner", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -170,4 +170,4 @@ def pretrained(name="camembert_base_token_classifier_wikiner", lang="en", remote The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CamemBertForTokenClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(CamemBertForTokenClassification, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/camembert_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/camembert_for_zero_shot_classification.py index 7b16c4475e5511..86564a358d9b4d 100644 --- a/python/sparknlp/annotator/classifier_dl/camembert_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/camembert_for_zero_shot_classification.py @@ -179,7 +179,7 @@ def loadSavedModel(folder, spark_session): return CamemBertForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="camembert_zero_shot_classifier_xnli_onnx", lang="fr", remote_loc=None): + def pretrained(name="camembert_zero_shot_classifier_xnli_onnx", lang="fr", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -199,4 +199,4 @@ def pretrained(name="camembert_zero_shot_classifier_xnli_onnx", lang="fr", remot The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CamemBertForZeroShotClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(CamemBertForZeroShotClassification, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/classifier_dl.py b/python/sparknlp/annotator/classifier_dl/classifier_dl.py index 90b30db9543a8d..fd3219c2437646 100755 --- a/python/sparknlp/annotator/classifier_dl/classifier_dl.py +++ b/python/sparknlp/annotator/classifier_dl/classifier_dl.py @@ -298,7 +298,7 @@ def setConfigProtoBytes(self, b): return self._set(configProtoBytes=b) @staticmethod - def pretrained(name="classifierdl_use_trec6", lang="en", remote_loc=None): + def pretrained(name="classifierdl_use_trec6", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -317,4 +317,4 @@ def pretrained(name="classifierdl_use_trec6", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ClassifierDLModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(ClassifierDLModel, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/deberta_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/deberta_for_question_answering.py index bc2c09c6a434a3..7424a86c62ddd4 100755 --- a/python/sparknlp/annotator/classifier_dl/deberta_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/deberta_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return DeBertaForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="deberta_v3_xsmall_qa_squad2", lang="en", remote_loc=None): + def pretrained(name="deberta_v3_xsmall_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="deberta_v3_xsmall_qa_squad2", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DeBertaForQuestionAnswering, name, lang, remote_loc) + return ResourceDownloader.downloadModel(DeBertaForQuestionAnswering, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/deberta_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/deberta_for_sequence_classification.py index 9ca03167f9ed7b..268888c791f30c 100755 --- a/python/sparknlp/annotator/classifier_dl/deberta_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/deberta_for_sequence_classification.py @@ -175,7 +175,7 @@ def loadSavedModel(folder, spark_session): return DeBertaForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="deberta_base_sequence_classifier_imdb", lang="en", remote_loc=None): + def pretrained(name="deberta_v3_xsmall_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -195,4 +195,4 @@ def pretrained(name="deberta_base_sequence_classifier_imdb", lang="en", remote_l The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DeBertaForSequenceClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(DeBertaForSequenceClassification, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/deberta_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/deberta_for_token_classification.py index 436118521414fd..60493c45f79dfe 100755 --- a/python/sparknlp/annotator/classifier_dl/deberta_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/deberta_for_token_classification.py @@ -152,7 +152,7 @@ def loadSavedModel(folder, spark_session): return DeBertaForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="deberta_v3_xsmall_token_classifier_conll03", lang="en", remote_loc=None): + def pretrained(name="deberta_v3_xsmall_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -172,4 +172,4 @@ def pretrained(name="deberta_v3_xsmall_token_classifier_conll03", lang="en", rem The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DeBertaForTokenClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(DeBertaForTokenClassification, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/deberta_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/deberta_for_zero_shot_classification.py index 6e8364cf8caaf4..c61d75684f2f57 100644 --- a/python/sparknlp/annotator/classifier_dl/deberta_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/deberta_for_zero_shot_classification.py @@ -172,7 +172,7 @@ def loadSavedModel(folder, spark_session): return DeBertaForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="deberta_base_zero_shot_classifier_mnli_anli_v3", lang="en", remote_loc=None): + def pretrained(name="deberta_base_zero_shot_classifier_mnli_anli_v3", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters ---------- @@ -190,4 +190,4 @@ def pretrained(name="deberta_base_zero_shot_classifier_mnli_anli_v3", lang="en", The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DeBertaForZeroShotClassification, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(DeBertaForZeroShotClassification, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/distil_bert_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/distil_bert_for_question_answering.py index 15230c58b69791..fe8bdd5c49254c 100755 --- a/python/sparknlp/annotator/classifier_dl/distil_bert_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/distil_bert_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return DistilBertForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_cased_qa_squad2", lang="en", remote_loc=None): + def pretrained(name="distilbert_base_cased_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="distilbert_base_cased_qa_squad2", lang="en", remote_loc=Non The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertForQuestionAnswering, name, lang, remote_loc) + return ResourceDownloader.downloadModel(DistilBertForQuestionAnswering, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/distil_bert_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/distil_bert_for_sequence_classification.py index 16a7222cb1b1a0..6aa1c861c27f0c 100755 --- a/python/sparknlp/annotator/classifier_dl/distil_bert_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/distil_bert_for_sequence_classification.py @@ -178,7 +178,7 @@ def loadSavedModel(folder, spark_session): return DistilBertForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_sequence_classifier_imdb", lang="en", remote_loc=None): + def pretrained(name="distilbert_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -198,4 +198,4 @@ def pretrained(name="distilbert_base_sequence_classifier_imdb", lang="en", remot The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertForSequenceClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(DistilBertForSequenceClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/distil_bert_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/distil_bert_for_token_classification.py index 984f5dfd732179..53a3d39b127086 100755 --- a/python/sparknlp/annotator/classifier_dl/distil_bert_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/distil_bert_for_token_classification.py @@ -152,7 +152,7 @@ def loadSavedModel(folder, spark_session): return DistilBertForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_token_classifier_conll03", lang="en", remote_loc=None): + def pretrained(name="distilbert_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -172,4 +172,4 @@ def pretrained(name="distilbert_base_token_classifier_conll03", lang="en", remot The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertForTokenClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(DistilBertForTokenClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/distil_bert_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/distil_bert_for_zero_shot_classification.py index 803179b46f2327..c93adc650f02fe 100644 --- a/python/sparknlp/annotator/classifier_dl/distil_bert_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/distil_bert_for_zero_shot_classification.py @@ -188,7 +188,7 @@ def loadSavedModel(folder, spark_session): return DistilBertForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_zero_shot_classifier_uncased_mnli", lang="en", remote_loc=None): + def pretrained(name="distilbert_base_zero_shot_classifier_uncased_mnli", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -208,4 +208,4 @@ def pretrained(name="distilbert_base_zero_shot_classifier_uncased_mnli", lang="e The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertForZeroShotClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(DistilBertForZeroShotClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/distilbert_for_multiple_choice.py b/python/sparknlp/annotator/classifier_dl/distilbert_for_multiple_choice.py index f76aa3859c307e..d6bcf925f0cfe4 100644 --- a/python/sparknlp/annotator/classifier_dl/distilbert_for_multiple_choice.py +++ b/python/sparknlp/annotator/classifier_dl/distilbert_for_multiple_choice.py @@ -138,7 +138,7 @@ def loadSavedModel(folder, spark_session): return DistilBertForMultipleChoice(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_uncased_multiple_choice", lang="en", remote_loc=None): + def pretrained(name="distilbert_base_uncased_multiple_choice", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -158,4 +158,4 @@ def pretrained(name="distilbert_base_uncased_multiple_choice", lang="en", remote The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertForMultipleChoice, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(DistilBertForMultipleChoice, name, lang, remote_loc,engine ) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/longformer_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/longformer_for_question_answering.py index c6f96e43e0480d..5b9052ae445409 100755 --- a/python/sparknlp/annotator/classifier_dl/longformer_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/longformer_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return LongformerForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="longformer_base_base_qa_squad2", lang="en", remote_loc=None): + def pretrained(name="longformer_base_base_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="longformer_base_base_qa_squad2", lang="en", remote_loc=None The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LongformerForQuestionAnswering, name, lang, remote_loc) + return ResourceDownloader.downloadModel(LongformerForQuestionAnswering, name, lang, remote_loc,engine ) diff --git a/python/sparknlp/annotator/classifier_dl/longformer_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/longformer_for_sequence_classification.py index 869ea11540a3af..24ee808c5fb057 100755 --- a/python/sparknlp/annotator/classifier_dl/longformer_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/longformer_for_sequence_classification.py @@ -178,7 +178,7 @@ def loadSavedModel(folder, spark_session): return LongformerForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="longformer_base_sequence_classifier_imdb", lang="en", remote_loc=None): + def pretrained(name="longformer_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -198,4 +198,4 @@ def pretrained(name="longformer_base_sequence_classifier_imdb", lang="en", remot The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LongformerForSequenceClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(LongformerForSequenceClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/longformer_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/longformer_for_token_classification.py index b9d4a4b21178ce..533d876a9fff60 100755 --- a/python/sparknlp/annotator/classifier_dl/longformer_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/longformer_for_token_classification.py @@ -153,7 +153,7 @@ def loadSavedModel(folder, spark_session): return LongformerForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="longformer_base_token_classifier_conll03", lang="en", remote_loc=None): + def pretrained(name="longformer_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -173,4 +173,4 @@ def pretrained(name="longformer_base_token_classifier_conll03", lang="en", remot The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LongformerForTokenClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(LongformerForTokenClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/mpnet_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/mpnet_for_question_answering.py index 1738ce0cfd7f8c..840a95efad9cc6 100755 --- a/python/sparknlp/annotator/classifier_dl/mpnet_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/mpnet_for_question_answering.py @@ -125,7 +125,7 @@ def loadSavedModel(folder, spark_session): return MPNetForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="mpnet_base_question_answering_squad2", lang="en", remote_loc=None): + def pretrained(name="mpnet_base_question_answering_squad2", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -145,4 +145,4 @@ def pretrained(name="mpnet_base_question_answering_squad2", lang="en", remote_lo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MPNetForQuestionAnswering, name, lang, remote_loc) + return ResourceDownloader.downloadModel(MPNetForQuestionAnswering, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/mpnet_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/mpnet_for_sequence_classification.py index 0f943ab16364fb..8c53f8d0ee3074 100755 --- a/python/sparknlp/annotator/classifier_dl/mpnet_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/mpnet_for_sequence_classification.py @@ -165,7 +165,7 @@ def loadSavedModel(folder, spark_session): return MPNetForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="mpnet_sequence_classifier_ukr_message", lang="en", remote_loc=None): + def pretrained(name="mpnet_sequence_classifier_ukr_message", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -185,4 +185,4 @@ def pretrained(name="mpnet_sequence_classifier_ukr_message", lang="en", remote_l The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MPNetForSequenceClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(MPNetForSequenceClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/mpnet_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/mpnet_for_token_classification.py index c3111e95dbf337..13a3d72055dafb 100644 --- a/python/sparknlp/annotator/classifier_dl/mpnet_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/mpnet_for_token_classification.py @@ -150,7 +150,7 @@ def loadSavedModel(folder, spark_session): return MPNetForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None): + def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -170,4 +170,4 @@ def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MPNetForTokenClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(MPNetForTokenClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/multi_classifier_dl.py b/python/sparknlp/annotator/classifier_dl/multi_classifier_dl.py index 51922a2afbd25f..3b0929a365aac4 100755 --- a/python/sparknlp/annotator/classifier_dl/multi_classifier_dl.py +++ b/python/sparknlp/annotator/classifier_dl/multi_classifier_dl.py @@ -372,7 +372,7 @@ def setConfigProtoBytes(self, b): return self._set(configProtoBytes=b) @staticmethod - def pretrained(name="multiclassifierdl_use_toxic", lang="en", remote_loc=None): + def pretrained(name="multiclassifierdl_use_toxic", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -392,4 +392,4 @@ def pretrained(name="multiclassifierdl_use_toxic", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MultiClassifierDLModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(MultiClassifierDLModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/roberta_for_multiple_choice.py b/python/sparknlp/annotator/classifier_dl/roberta_for_multiple_choice.py index 4e01000b83b7dd..74aa17c446a3b4 100644 --- a/python/sparknlp/annotator/classifier_dl/roberta_for_multiple_choice.py +++ b/python/sparknlp/annotator/classifier_dl/roberta_for_multiple_choice.py @@ -138,7 +138,7 @@ def loadSavedModel(folder, spark_session): return RoBertaForMultipleChoice(java_model=jModel) @staticmethod - def pretrained(name="Roberta_base_uncased_multiple_choice", lang="en", remote_loc=None): + def pretrained(name="Roberta_base_uncased_multiple_choice", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -158,4 +158,4 @@ def pretrained(name="Roberta_base_uncased_multiple_choice", lang="en", remote_lo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaForMultipleChoice, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(RoBertaForMultipleChoice, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/roberta_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/roberta_for_question_answering.py index 27a6bfdb979ef4..9fcaa3f0f32d59 100755 --- a/python/sparknlp/annotator/classifier_dl/roberta_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/roberta_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return RoBertaForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="roberta_base_qa_squad2", lang="en", remote_loc=None): + def pretrained(name="roberta_base_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="roberta_base_qa_squad2", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaForQuestionAnswering, name, lang, remote_loc) + return ResourceDownloader.downloadModel(RoBertaForQuestionAnswering, name, lang, remote_loc, engine) diff --git a/python/sparknlp/annotator/classifier_dl/roberta_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/roberta_for_sequence_classification.py index 6b26347c4397ca..13cb43228b55f1 100755 --- a/python/sparknlp/annotator/classifier_dl/roberta_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/roberta_for_sequence_classification.py @@ -178,7 +178,7 @@ def loadSavedModel(folder, spark_session): return RoBertaForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="roberta_base_sequence_classifier_imdb", lang="en", remote_loc=None): + def pretrained(name="roberta_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -198,4 +198,4 @@ def pretrained(name="roberta_base_sequence_classifier_imdb", lang="en", remote_l The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaForSequenceClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(RoBertaForSequenceClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/roberta_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/roberta_for_token_classification.py index 6cc071a29092b0..c8fdc4ac6a99af 100755 --- a/python/sparknlp/annotator/classifier_dl/roberta_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/roberta_for_token_classification.py @@ -166,7 +166,7 @@ def loadSavedModel(folder, spark_session): return RoBertaForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="roberta_base_token_classifier_conll03", lang="en", remote_loc=None): + def pretrained(name="roberta_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -186,4 +186,4 @@ def pretrained(name="roberta_base_token_classifier_conll03", lang="en", remote_l The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaForTokenClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(RoBertaForTokenClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/roberta_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/roberta_for_zero_shot_classification.py index 29097acd25c553..bd26088ba1f92d 100644 --- a/python/sparknlp/annotator/classifier_dl/roberta_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/roberta_for_zero_shot_classification.py @@ -202,7 +202,7 @@ def loadSavedModel(folder, spark_session): return RoBertaForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="roberta_base_zero_shot_classifier_nli", lang="en", remote_loc=None): + def pretrained(name="roberta_base_zero_shot_classifier_nli", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -222,4 +222,4 @@ def pretrained(name="roberta_base_zero_shot_classifier_nli", lang="en", remote_l The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaForZeroShotClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(RoBertaForZeroShotClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/sentiment_dl.py b/python/sparknlp/annotator/classifier_dl/sentiment_dl.py index 362df774e6d81a..6c2c533dd0c482 100755 --- a/python/sparknlp/annotator/classifier_dl/sentiment_dl.py +++ b/python/sparknlp/annotator/classifier_dl/sentiment_dl.py @@ -356,7 +356,7 @@ def setThresholdLabel(self, p): return self._set(thresholdLabel=p) @staticmethod - def pretrained(name="sentimentdl_use_imdb", lang="en", remote_loc=None): + def pretrained(name="sentimentdl_use_imdb", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -375,4 +375,4 @@ def pretrained(name="sentimentdl_use_imdb", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SentimentDLModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(SentimentDLModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/tapas_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/tapas_for_question_answering.py index 5911d94d2f92e2..12d84d2146d182 100644 --- a/python/sparknlp/annotator/classifier_dl/tapas_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/tapas_for_question_answering.py @@ -147,7 +147,7 @@ def loadSavedModel(folder, spark_session): return TapasForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="table_qa_tapas_base_finetuned_wtq", lang="en", remote_loc=None): + def pretrained(name="table_qa_tapas_base_finetuned_wtq", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -167,4 +167,4 @@ def pretrained(name="table_qa_tapas_base_finetuned_wtq", lang="en", remote_loc=N The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(TapasForQuestionAnswering, name, lang, remote_loc) + return ResourceDownloader.downloadModel(TapasForQuestionAnswering, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_multiple_choice.py b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_multiple_choice.py index 8da691d35cc091..88ba63d0f6ad14 100644 --- a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_multiple_choice.py +++ b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_multiple_choice.py @@ -126,7 +126,7 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaForMultipleChoice(java_model=jModel) @staticmethod - def pretrained(name="xlm_roberta_base_mc", lang="en", remote_loc=None): + def pretrained(name="xlm_roberta_base_mc", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -146,4 +146,4 @@ def pretrained(name="xlm_roberta_base_mc", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaForMultipleChoice, name, lang, remote_loc) + return ResourceDownloader.downloadModel(XlmRoBertaForMultipleChoice, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_question_answering.py index 97a2a38c08df86..117977cf7fe58c 100755 --- a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="xlm_roberta_base_qa_squad2", lang="en", remote_loc=None): + def pretrained(name="xlm_roberta_base_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="xlm_roberta_base_qa_squad2", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaForQuestionAnswering, name, lang, remote_loc) + return ResourceDownloader.downloadModel(XlmRoBertaForQuestionAnswering, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_sequence_classification.py index 2db4b3b7ae7b2a..b0ce95801db775 100755 --- a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_sequence_classification.py @@ -178,14 +178,14 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="xlm_roberta_base_sequence_classifier_imdb", lang="en", remote_loc=None): + def pretrained(name="xlm_roberta_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters ---------- name : str, optional Name of the pretrained model, by default - "xlm_roberta_base_sequence_classifier_imdb" + "xlm_roberta_base_token_classifier_conll03" lang : str, optional Language of the pretrained model, by default "en" remote_loc : str, optional @@ -198,4 +198,4 @@ def pretrained(name="xlm_roberta_base_sequence_classifier_imdb", lang="en", remo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaForSequenceClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(XlmRoBertaForSequenceClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_token_classification.py index 50a3668a181cea..93c66592829367 100755 --- a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_token_classification.py @@ -150,7 +150,7 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None): + def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -170,4 +170,4 @@ def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaForTokenClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(XlmRoBertaForTokenClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_zero_shot_classification.py index 36670c155a2d14..1cb1c10b713287 100644 --- a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_zero_shot_classification.py @@ -202,7 +202,7 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="xlm_roberta_large_zero_shot_classifier_xnli_anli", lang="xx", remote_loc=None): + def pretrained(name="xlm_roberta_large_zero_shot_classifier_xnli_anli", lang="xx", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -222,4 +222,4 @@ def pretrained(name="xlm_roberta_large_zero_shot_classifier_xnli_anli", lang="xx The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaForZeroShotClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(XlmRoBertaForZeroShotClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/xlnet_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/xlnet_for_sequence_classification.py index 8a6c6ce5111e16..195f55a4af5b1e 100755 --- a/python/sparknlp/annotator/classifier_dl/xlnet_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/xlnet_for_sequence_classification.py @@ -178,7 +178,7 @@ def loadSavedModel(folder, spark_session): return XlnetForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="xlnet_base_sequence_classifier_imdb", lang="en", remote_loc=None): + def pretrained(name="xlnet_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -198,4 +198,4 @@ def pretrained(name="xlnet_base_sequence_classifier_imdb", lang="en", remote_loc The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlnetForSequenceClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(XlnetForSequenceClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/classifier_dl/xlnet_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/xlnet_for_token_classification.py index e0d1730b557a5b..01ab9a0807ea0d 100755 --- a/python/sparknlp/annotator/classifier_dl/xlnet_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/xlnet_for_token_classification.py @@ -153,7 +153,7 @@ def loadSavedModel(folder, spark_session): return XlnetForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="xlnet_base_token_classifier_conll03", lang="en", remote_loc=None): + def pretrained(name="xlnet_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -173,4 +173,4 @@ def pretrained(name="xlnet_base_token_classifier_conll03", lang="en", remote_loc The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlnetForTokenClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(XlnetForTokenClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/coref/spanbert_coref.py b/python/sparknlp/annotator/coref/spanbert_coref.py index b0d9d4761e42e0..07d9b25fcc617b 100644 --- a/python/sparknlp/annotator/coref/spanbert_coref.py +++ b/python/sparknlp/annotator/coref/spanbert_coref.py @@ -199,7 +199,7 @@ def loadSavedModel(folder, spark_session): return SpanBertCorefModel(java_model=jModel) @staticmethod - def pretrained(name="spanbert_base_coref", lang="en", remote_loc=None): + def pretrained(name="spanbert_base_coref", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -218,4 +218,4 @@ def pretrained(name="spanbert_base_coref", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SpanBertCorefModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(SpanBertCorefModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/cv/blip_for_question_answering.py b/python/sparknlp/annotator/cv/blip_for_question_answering.py index fe018c0e683bf2..98b54b9325fe5b 100644 --- a/python/sparknlp/annotator/cv/blip_for_question_answering.py +++ b/python/sparknlp/annotator/cv/blip_for_question_answering.py @@ -149,7 +149,7 @@ def loadSavedModel(folder, spark_session): return BLIPForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="blip_vqa_base", lang="en", remote_loc=None): + def pretrained(name="blip_vqa_base", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -169,4 +169,4 @@ def pretrained(name="blip_vqa_base", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BLIPForQuestionAnswering, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(BLIPForQuestionAnswering, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/clip_for_zero_shot_classification.py b/python/sparknlp/annotator/cv/clip_for_zero_shot_classification.py index 775f61f7d4d9bf..836a078b79982e 100644 --- a/python/sparknlp/annotator/cv/clip_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/cv/clip_for_zero_shot_classification.py @@ -170,7 +170,7 @@ def loadSavedModel(folder, spark_session): return CLIPForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="zero_shot_classifier_clip_vit_base_patch32", lang="en", remote_loc=None): + def pretrained(name="zero_shot_classifier_clip_vit_base_patch32", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -190,4 +190,4 @@ def pretrained(name="zero_shot_classifier_clip_vit_base_patch32", lang="en", rem The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CLIPForZeroShotClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(CLIPForZeroShotClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/cv/convnext_for_image_classification.py b/python/sparknlp/annotator/cv/convnext_for_image_classification.py index d86b71a3e44899..a2951a69447dad 100644 --- a/python/sparknlp/annotator/cv/convnext_for_image_classification.py +++ b/python/sparknlp/annotator/cv/convnext_for_image_classification.py @@ -245,7 +245,7 @@ def loadSavedModel(folder, spark_session): @staticmethod def pretrained(name="image_classifier_convnext_tiny_224_local", lang="en", - remote_loc=None): + remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -266,4 +266,4 @@ def pretrained(name="image_classifier_convnext_tiny_224_local", lang="en", """ from sparknlp.pretrained import ResourceDownloader return ResourceDownloader.downloadModel(ConvNextForImageClassification, name, lang, - remote_loc) + remote_loc,engine) diff --git a/python/sparknlp/annotator/cv/florence2_transformer.py b/python/sparknlp/annotator/cv/florence2_transformer.py index 0ba21c707823d6..e79882eef1131d 100644 --- a/python/sparknlp/annotator/cv/florence2_transformer.py +++ b/python/sparknlp/annotator/cv/florence2_transformer.py @@ -174,7 +174,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Florence2Transformer(java_model=jModel) @staticmethod - def pretrained(name="florence2_base_ft_int4", lang="en", remote_loc=None): + def pretrained(name="florence2_base_ft_int4", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model.""" from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Florence2Transformer, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(Florence2Transformer, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/gemma3_for_multimodal.py b/python/sparknlp/annotator/cv/gemma3_for_multimodal.py index 6d6b8e5156adec..cbb3722503cde2 100644 --- a/python/sparknlp/annotator/cv/gemma3_for_multimodal.py +++ b/python/sparknlp/annotator/cv/gemma3_for_multimodal.py @@ -324,7 +324,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Gemma3ForMultiModal(java_model=jModel) @staticmethod - def pretrained(name="gemma3_4b_it_int4", lang="en", remote_loc=None): + def pretrained(name="gemma_3_4b_it_int4", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -343,4 +343,4 @@ def pretrained(name="gemma3_4b_it_int4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Gemma3ForMultiModal, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(Gemma3ForMultiModal, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/internvl_for_multimodal.py b/python/sparknlp/annotator/cv/internvl_for_multimodal.py index a39476f5558aa7..2dcaefe81dedf5 100644 --- a/python/sparknlp/annotator/cv/internvl_for_multimodal.py +++ b/python/sparknlp/annotator/cv/internvl_for_multimodal.py @@ -259,7 +259,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return InternVLForMultiModal(java_model=jModel) @staticmethod - def pretrained(name="internvl2_5_1b_int4", lang="en", remote_loc=None): + def pretrained(name="internvl2_5_1b_int4", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters ---------- @@ -277,4 +277,4 @@ def pretrained(name="internvl2_5_1b_int4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(InternVLForMultiModal, name, lang, remote_loc) + return ResourceDownloader.downloadModel(InternVLForMultiModal, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/cv/janus_for_multimodal.py b/python/sparknlp/annotator/cv/janus_for_multimodal.py index 0e70cfe0c3edc8..cc998241c5c72c 100644 --- a/python/sparknlp/annotator/cv/janus_for_multimodal.py +++ b/python/sparknlp/annotator/cv/janus_for_multimodal.py @@ -330,7 +330,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return JanusForMultiModal(java_model=jModel) @staticmethod - def pretrained(name="janus_1_3b_int4", lang="en", remote_loc=None): + def pretrained(name="janus_1_3b_int4", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters ---------- @@ -348,4 +348,4 @@ def pretrained(name="janus_1_3b_int4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(JanusForMultiModal, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(JanusForMultiModal, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/llava_for_multimodal.py b/python/sparknlp/annotator/cv/llava_for_multimodal.py index 84050df19806c4..caa94539e93975 100644 --- a/python/sparknlp/annotator/cv/llava_for_multimodal.py +++ b/python/sparknlp/annotator/cv/llava_for_multimodal.py @@ -305,7 +305,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return LLAVAForMultiModal(java_model=jModel) @staticmethod - def pretrained(name="llava_1_5_7b_hf", lang="en", remote_loc=None): + def pretrained(name="llava_1_5_7b_hf", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -325,4 +325,4 @@ def pretrained(name="llava_1_5_7b_hf", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LLAVAForMultiModal, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(LLAVAForMultiModal, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/mllama_for_multimodal.py b/python/sparknlp/annotator/cv/mllama_for_multimodal.py index 1a4939b739d957..619eb1fb0d534b 100644 --- a/python/sparknlp/annotator/cv/mllama_for_multimodal.py +++ b/python/sparknlp/annotator/cv/mllama_for_multimodal.py @@ -317,7 +317,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return MLLamaForMultimodal(java_model=jModel) @staticmethod - def pretrained(name="llama_3_2_11b_vision_instruct_int4", lang="en", remote_loc=None): + def pretrained(name="llama_3_2_11b_vision_instruct_int4", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -337,4 +337,4 @@ def pretrained(name="llama_3_2_11b_vision_instruct_int4", lang="en", remote_loc= The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MLLamaForMultimodal, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(MLLamaForMultimodal, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/paligemma_for_multimodal.py b/python/sparknlp/annotator/cv/paligemma_for_multimodal.py index 06537b6ce805dc..4dabf65cafe154 100644 --- a/python/sparknlp/annotator/cv/paligemma_for_multimodal.py +++ b/python/sparknlp/annotator/cv/paligemma_for_multimodal.py @@ -284,7 +284,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return PaliGemmaForMultiModal(java_model=jModel) @staticmethod - def pretrained(name="paligemma_3b_pt_224_int4", lang="en", remote_loc=None): + def pretrained(name="paligemma_3b_pt_224_int4", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -304,5 +304,5 @@ def pretrained(name="paligemma_3b_pt_224_int4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(PaliGemmaForMultiModal, name, lang, remote_loc) + return ResourceDownloader.downloadModel(PaliGemmaForMultiModal, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/cv/phi3_vision_for_multimodal.py b/python/sparknlp/annotator/cv/phi3_vision_for_multimodal.py index 55d3c8c7b1c7fd..6fc6a87cfb8b9b 100644 --- a/python/sparknlp/annotator/cv/phi3_vision_for_multimodal.py +++ b/python/sparknlp/annotator/cv/phi3_vision_for_multimodal.py @@ -305,7 +305,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Phi3Vision(java_model=jModel) @staticmethod - def pretrained(name="phi_3_vision_128k_instruct", lang="en", remote_loc=None): + def pretrained(name="phi_3_vision_128k_instruct", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -325,4 +325,4 @@ def pretrained(name="phi_3_vision_128k_instruct", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Phi3Vision, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(Phi3Vision, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/qwen2vl_transformer.py b/python/sparknlp/annotator/cv/qwen2vl_transformer.py index 51433aed7cc7e4..8aab66e8cd0d30 100644 --- a/python/sparknlp/annotator/cv/qwen2vl_transformer.py +++ b/python/sparknlp/annotator/cv/qwen2vl_transformer.py @@ -309,7 +309,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Qwen2VLTransformer(java_model=jModel) @staticmethod - def pretrained(name="qwen2_vl_2b_instruct_int4", lang="en", remote_loc=None): + def pretrained(name="qwen2_vl_2b_instruct_int4", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -329,4 +329,4 @@ def pretrained(name="qwen2_vl_2b_instruct_int4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Qwen2VLTransformer, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(Qwen2VLTransformer, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/smolvlm_transformer.py b/python/sparknlp/annotator/cv/smolvlm_transformer.py index 461dcc84dcb6b9..7f79d4aaa92b13 100644 --- a/python/sparknlp/annotator/cv/smolvlm_transformer.py +++ b/python/sparknlp/annotator/cv/smolvlm_transformer.py @@ -405,7 +405,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return SmolVLMTransformer(java_model=jModel) @staticmethod - def pretrained(name="smolvlm_instruct_int4", lang="en", remote_loc=None): + def pretrained(name="smolvlm_instruct_int4", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters ---------- @@ -423,4 +423,4 @@ def pretrained(name="smolvlm_instruct_int4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SmolVLMTransformer, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(SmolVLMTransformer, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/swin_for_image_classification.py b/python/sparknlp/annotator/cv/swin_for_image_classification.py index 958b8e4a2b8300..f0997eacc6f96d 100644 --- a/python/sparknlp/annotator/cv/swin_for_image_classification.py +++ b/python/sparknlp/annotator/cv/swin_for_image_classification.py @@ -218,7 +218,7 @@ def loadSavedModel(folder, spark_session): @staticmethod def pretrained(name="image_classifier_swin_base_patch4_window7_224", lang="en", - remote_loc=None): + remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -239,4 +239,4 @@ def pretrained(name="image_classifier_swin_base_patch4_window7_224", lang="en", """ from sparknlp.pretrained import ResourceDownloader return ResourceDownloader.downloadModel(SwinForImageClassification, name, lang, - remote_loc) + remote_loc,engine) diff --git a/python/sparknlp/annotator/cv/vision_encoder_decoder_for_image_captioning.py b/python/sparknlp/annotator/cv/vision_encoder_decoder_for_image_captioning.py index 6fe6f81b5c9012..4f44b950795a18 100644 --- a/python/sparknlp/annotator/cv/vision_encoder_decoder_for_image_captioning.py +++ b/python/sparknlp/annotator/cv/vision_encoder_decoder_for_image_captioning.py @@ -217,7 +217,7 @@ def loadSavedModel(folder, spark_session): return VisionEncoderDecoderForImageCaptioning(java_model=jModel) @staticmethod - def pretrained(name="image_captioning_vit_gpt2", lang="en", remote_loc=None): + def pretrained(name="image_captioning_vit_gpt2", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -237,4 +237,4 @@ def pretrained(name="image_captioning_vit_gpt2", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(VisionEncoderDecoderForImageCaptioning, name, lang, remote_loc) + return ResourceDownloader.downloadModel(VisionEncoderDecoderForImageCaptioning, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/cv/vit_for_image_classification.py b/python/sparknlp/annotator/cv/vit_for_image_classification.py index 7ad99661e9b19b..6d405d6e39ef9f 100644 --- a/python/sparknlp/annotator/cv/vit_for_image_classification.py +++ b/python/sparknlp/annotator/cv/vit_for_image_classification.py @@ -194,7 +194,7 @@ def loadSavedModel(folder, spark_session): return ViTForImageClassification(java_model=jModel) @staticmethod - def pretrained(name="image_classifier_vit_base_patch16_224", lang="en", remote_loc=None): + def pretrained(name="image_classifier_vit_base_patch16_224", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model. Parameters @@ -214,4 +214,4 @@ def pretrained(name="image_classifier_vit_base_patch16_224", lang="en", remote_l The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ViTForImageClassification, name, lang, remote_loc) + return ResourceDownloader.downloadModel(ViTForImageClassification, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/dependency/dependency_parser.py b/python/sparknlp/annotator/dependency/dependency_parser.py index 6a0789ae5e416f..9ddc39bab60ea4 100755 --- a/python/sparknlp/annotator/dependency/dependency_parser.py +++ b/python/sparknlp/annotator/dependency/dependency_parser.py @@ -272,7 +272,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.parser.dep.Depende ) @staticmethod - def pretrained(name="dependency_conllu", lang="en", remote_loc=None): + def pretrained(name="dependency_conllu", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -291,4 +291,4 @@ def pretrained(name="dependency_conllu", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DependencyParserModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(DependencyParserModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/dependency/typed_dependency_parser.py b/python/sparknlp/annotator/dependency/typed_dependency_parser.py index 8fb0476c29bd18..990edeb913bab9 100755 --- a/python/sparknlp/annotator/dependency/typed_dependency_parser.py +++ b/python/sparknlp/annotator/dependency/typed_dependency_parser.py @@ -296,7 +296,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.parser.typdep.Type ) @staticmethod - def pretrained(name="dependency_typed_conllu", lang="en", remote_loc=None): + def pretrained(name="dependency_typed_conllu", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -315,4 +315,4 @@ def pretrained(name="dependency_typed_conllu", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(TypedDependencyParserModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(TypedDependencyParserModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/albert_embeddings.py b/python/sparknlp/annotator/embeddings/albert_embeddings.py index 72eef1e82d87e4..a87d488c627465 100755 --- a/python/sparknlp/annotator/embeddings/albert_embeddings.py +++ b/python/sparknlp/annotator/embeddings/albert_embeddings.py @@ -208,7 +208,7 @@ def loadSavedModel(folder, spark_session): return AlbertEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="albert_base_uncased", lang="en", remote_loc=None): + def pretrained(name="albert_base_uncased", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -227,4 +227,4 @@ def pretrained(name="albert_base_uncased", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(AlbertEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/auto_gguf_embeddings.py b/python/sparknlp/annotator/embeddings/auto_gguf_embeddings.py index ef0d8aabd78601..8604900846107f 100755 --- a/python/sparknlp/annotator/embeddings/auto_gguf_embeddings.py +++ b/python/sparknlp/annotator/embeddings/auto_gguf_embeddings.py @@ -509,7 +509,7 @@ def loadSavedModel(folder, spark_session): return AutoGGUFEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="Qwen3_Embedding_0.6B_Q8_0_gguf", lang="en", remote_loc=None): + def pretrained(name="Qwen3_Embedding_0.6B_Q8_0_gguf", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -530,7 +530,7 @@ def pretrained(name="Qwen3_Embedding_0.6B_Q8_0_gguf", lang="en", remote_loc=None from sparknlp.pretrained import ResourceDownloader return ResourceDownloader.downloadModel( - AutoGGUFEmbeddings, name, lang, remote_loc + AutoGGUFEmbeddings, name, lang, remote_loc,engine ) def close(self): diff --git a/python/sparknlp/annotator/embeddings/bert_embeddings.py b/python/sparknlp/annotator/embeddings/bert_embeddings.py index 9fb7b40076ef28..df5fde7824bb58 100755 --- a/python/sparknlp/annotator/embeddings/bert_embeddings.py +++ b/python/sparknlp/annotator/embeddings/bert_embeddings.py @@ -186,7 +186,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return BertEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="small_bert_L2_768", lang="en", remote_loc=None): + def pretrained(name="small_bert_L2_768", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -205,4 +205,4 @@ def pretrained(name="small_bert_L2_768", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(BertEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/bert_sentence_embeddings.py b/python/sparknlp/annotator/embeddings/bert_sentence_embeddings.py index c2545620bfb97f..05e84e6d27a142 100755 --- a/python/sparknlp/annotator/embeddings/bert_sentence_embeddings.py +++ b/python/sparknlp/annotator/embeddings/bert_sentence_embeddings.py @@ -202,7 +202,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return BertSentenceEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="sent_small_bert_L2_768", lang="en", remote_loc=None): + def pretrained(name="sent_small_bert_L2_768", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -221,4 +221,4 @@ def pretrained(name="sent_small_bert_L2_768", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertSentenceEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(BertSentenceEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/bge_embeddings.py b/python/sparknlp/annotator/embeddings/bge_embeddings.py index 555d3169a234ea..9d5b2a5613ab02 100644 --- a/python/sparknlp/annotator/embeddings/bge_embeddings.py +++ b/python/sparknlp/annotator/embeddings/bge_embeddings.py @@ -177,7 +177,7 @@ def loadSavedModel(folder, spark_session): return BGEEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="bge_small_en_v1.5", lang="en", remote_loc=None): + def pretrained(name="bge_small_en_v1.5", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -196,4 +196,4 @@ def pretrained(name="bge_small_en_v1.5", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BGEEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(BGEEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/camembert_embeddings.py b/python/sparknlp/annotator/embeddings/camembert_embeddings.py index 2461324b143b23..b7112d4df7daaf 100755 --- a/python/sparknlp/annotator/embeddings/camembert_embeddings.py +++ b/python/sparknlp/annotator/embeddings/camembert_embeddings.py @@ -188,7 +188,7 @@ def loadSavedModel(folder, spark_session): return CamemBertEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="camembert_base", lang="fr", remote_loc=None): + def pretrained(name="camembert_base", lang="fr", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -207,4 +207,4 @@ def pretrained(name="camembert_base", lang="fr", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CamemBertEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(CamemBertEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/deberta_embeddings.py b/python/sparknlp/annotator/embeddings/deberta_embeddings.py index 24462a0022b724..dd218d6cc7a668 100755 --- a/python/sparknlp/annotator/embeddings/deberta_embeddings.py +++ b/python/sparknlp/annotator/embeddings/deberta_embeddings.py @@ -186,7 +186,7 @@ def loadSavedModel(folder, spark_session): return DeBertaEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="deberta_v3_base", lang="en", remote_loc=None): + def pretrained(name="deberta_v3_base", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -205,4 +205,4 @@ def pretrained(name="deberta_v3_base", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DeBertaEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(DeBertaEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/distil_bert_embeddings.py b/python/sparknlp/annotator/embeddings/distil_bert_embeddings.py index 6ae825aec01fde..31c6ef3e640177 100755 --- a/python/sparknlp/annotator/embeddings/distil_bert_embeddings.py +++ b/python/sparknlp/annotator/embeddings/distil_bert_embeddings.py @@ -199,7 +199,7 @@ def loadSavedModel(folder, spark_session): return DistilBertEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_cased", lang="en", remote_loc=None): + def pretrained(name="distilbert_base_cased", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -218,4 +218,4 @@ def pretrained(name="distilbert_base_cased", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(DistilBertEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/doc2vec.py b/python/sparknlp/annotator/embeddings/doc2vec.py index 1bc6c7120b8e77..82aa3b927e504e 100755 --- a/python/sparknlp/annotator/embeddings/doc2vec.py +++ b/python/sparknlp/annotator/embeddings/doc2vec.py @@ -323,7 +323,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.embeddings.Doc2VecModel", jav ) @staticmethod - def pretrained(name="doc2vec_gigaword_300", lang="en", remote_loc=None): + def pretrained(name="doc2vec_gigaword_300", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -342,7 +342,7 @@ def pretrained(name="doc2vec_gigaword_300", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Doc2VecModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(Doc2VecModel, name, lang, remote_loc,engine) def getVectors(self): """ diff --git a/python/sparknlp/annotator/embeddings/e5_embeddings.py b/python/sparknlp/annotator/embeddings/e5_embeddings.py index c6c9e7c0e20033..57d55bef78fd2a 100644 --- a/python/sparknlp/annotator/embeddings/e5_embeddings.py +++ b/python/sparknlp/annotator/embeddings/e5_embeddings.py @@ -173,7 +173,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return E5Embeddings(java_model=jModel) @staticmethod - def pretrained(name="e5_small", lang="en", remote_loc=None): + def pretrained(name="e5_small", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -192,4 +192,4 @@ def pretrained(name="e5_small", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(E5Embeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(E5Embeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/e5v_embeddings.py b/python/sparknlp/annotator/embeddings/e5v_embeddings.py index e8ee518a40333e..a66be3ea414b79 100644 --- a/python/sparknlp/annotator/embeddings/e5v_embeddings.py +++ b/python/sparknlp/annotator/embeddings/e5v_embeddings.py @@ -117,7 +117,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return E5VEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="e5v_int4", lang="en", remote_loc=None): + def pretrained(name="e5v_int4", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -135,4 +135,4 @@ def pretrained(name="e5v_int4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(E5VEmbeddings, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(E5VEmbeddings, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/embeddings/elmo_embeddings.py b/python/sparknlp/annotator/embeddings/elmo_embeddings.py index 46b9858e84f697..f1a958abb314cb 100755 --- a/python/sparknlp/annotator/embeddings/elmo_embeddings.py +++ b/python/sparknlp/annotator/embeddings/elmo_embeddings.py @@ -229,7 +229,7 @@ def loadSavedModel(folder, spark_session): return ElmoEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="elmo", lang="en", remote_loc=None): + def pretrained(name="elmo", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -248,4 +248,4 @@ def pretrained(name="elmo", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ElmoEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(ElmoEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/instructor_embeddings.py b/python/sparknlp/annotator/embeddings/instructor_embeddings.py index 31ca3c7fd52723..9ddbc19b4ead7f 100755 --- a/python/sparknlp/annotator/embeddings/instructor_embeddings.py +++ b/python/sparknlp/annotator/embeddings/instructor_embeddings.py @@ -182,7 +182,7 @@ def loadSavedModel(folder, spark_session): return InstructorEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="instructor_base", lang="en", remote_loc=None): + def pretrained(name="instructor_base", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -201,4 +201,4 @@ def pretrained(name="instructor_base", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(InstructorEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(InstructorEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/longformer_embeddings.py b/python/sparknlp/annotator/embeddings/longformer_embeddings.py index ee286a8b3ed0f9..02c027345bd3bf 100755 --- a/python/sparknlp/annotator/embeddings/longformer_embeddings.py +++ b/python/sparknlp/annotator/embeddings/longformer_embeddings.py @@ -189,7 +189,7 @@ def loadSavedModel(folder, spark_session): return LongformerEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="longformer_base_4096", lang="en", remote_loc=None): + def pretrained(name="longformer_base_4096", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -208,4 +208,4 @@ def pretrained(name="longformer_base_4096", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LongformerEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(LongformerEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/minilm_embeddings.py b/python/sparknlp/annotator/embeddings/minilm_embeddings.py index a569dfcef9e83c..870bf1cdd1a335 100644 --- a/python/sparknlp/annotator/embeddings/minilm_embeddings.py +++ b/python/sparknlp/annotator/embeddings/minilm_embeddings.py @@ -167,7 +167,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return MiniLMEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="minilm_l6_v2", lang="en", remote_loc=None): + def pretrained(name="minilm_l6_v2", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -186,4 +186,4 @@ def pretrained(name="minilm_l6_v2", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MiniLMEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(MiniLMEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/mpnet_embeddings.py b/python/sparknlp/annotator/embeddings/mpnet_embeddings.py index 0d72cf3c388319..9a3aa8a08ef198 100644 --- a/python/sparknlp/annotator/embeddings/mpnet_embeddings.py +++ b/python/sparknlp/annotator/embeddings/mpnet_embeddings.py @@ -170,7 +170,7 @@ def loadSavedModel(folder, spark_session): return MPNetEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="all_mpnet_base_v2", lang="en", remote_loc=None): + def pretrained(name="all_mpnet_base_v2", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -189,4 +189,4 @@ def pretrained(name="all_mpnet_base_v2", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MPNetEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(MPNetEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/mxbai_embeddings.py b/python/sparknlp/annotator/embeddings/mxbai_embeddings.py index 854e8cac4e7f39..7cc35e81c9a9c6 100644 --- a/python/sparknlp/annotator/embeddings/mxbai_embeddings.py +++ b/python/sparknlp/annotator/embeddings/mxbai_embeddings.py @@ -162,7 +162,7 @@ def loadSavedModel(folder, spark_session): return MxbaiEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="mxbai_large_v1", lang="en", remote_loc=None): + def pretrained(name="mxbai_large_v1", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -181,4 +181,4 @@ def pretrained(name="mxbai_large_v1", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MxbaiEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(MxbaiEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/nomic_embeddings.py b/python/sparknlp/annotator/embeddings/nomic_embeddings.py index b80597cac937d1..f3fd9be9acc87a 100644 --- a/python/sparknlp/annotator/embeddings/nomic_embeddings.py +++ b/python/sparknlp/annotator/embeddings/nomic_embeddings.py @@ -159,7 +159,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return NomicEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="nomic_embed_v1", lang="en", remote_loc=None): + def pretrained(name="nomic_embed_v1", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -178,4 +178,4 @@ def pretrained(name="nomic_embed_v1", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(NomicEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(NomicEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/roberta_embeddings.py b/python/sparknlp/annotator/embeddings/roberta_embeddings.py index 72d180ae18cce5..865c448b471534 100755 --- a/python/sparknlp/annotator/embeddings/roberta_embeddings.py +++ b/python/sparknlp/annotator/embeddings/roberta_embeddings.py @@ -203,7 +203,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return RoBertaEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="roberta_base", lang="en", remote_loc=None): + def pretrained(name="roberta_base", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -222,4 +222,4 @@ def pretrained(name="roberta_base", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(RoBertaEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/roberta_sentence_embeddings.py b/python/sparknlp/annotator/embeddings/roberta_sentence_embeddings.py index 8a6d02c245af40..25926642cdce4f 100755 --- a/python/sparknlp/annotator/embeddings/roberta_sentence_embeddings.py +++ b/python/sparknlp/annotator/embeddings/roberta_sentence_embeddings.py @@ -169,7 +169,7 @@ def loadSavedModel(folder, spark_session): return RoBertaSentenceEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="sent_roberta_base", lang="en", remote_loc=None): + def pretrained(name="sent_roberta_base", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -188,4 +188,4 @@ def pretrained(name="sent_roberta_base", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaSentenceEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(RoBertaSentenceEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/snowflake_embeddings.py b/python/sparknlp/annotator/embeddings/snowflake_embeddings.py index 2e13cd2596d6ab..cdfaa3d66ad45a 100644 --- a/python/sparknlp/annotator/embeddings/snowflake_embeddings.py +++ b/python/sparknlp/annotator/embeddings/snowflake_embeddings.py @@ -180,7 +180,7 @@ def loadSavedModel(folder, spark_session): return SnowFlakeEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="snowflake_artic_m", lang="en", remote_loc=None): + def pretrained(name="snowflake_artic_m", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -199,4 +199,4 @@ def pretrained(name="snowflake_artic_m", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SnowFlakeEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(SnowFlakeEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/uae_embeddings.py b/python/sparknlp/annotator/embeddings/uae_embeddings.py index 1a25d63ded20c8..ea3e92ffe7267d 100644 --- a/python/sparknlp/annotator/embeddings/uae_embeddings.py +++ b/python/sparknlp/annotator/embeddings/uae_embeddings.py @@ -189,7 +189,7 @@ def loadSavedModel(folder, spark_session): return UAEEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="uae_large_v1", lang="en", remote_loc=None): + def pretrained(name="uae_large_v1", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -208,4 +208,4 @@ def pretrained(name="uae_large_v1", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(UAEEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(UAEEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/universal_sentence_encoder.py b/python/sparknlp/annotator/embeddings/universal_sentence_encoder.py index ae155bba436f8d..87956015d8ffc5 100755 --- a/python/sparknlp/annotator/embeddings/universal_sentence_encoder.py +++ b/python/sparknlp/annotator/embeddings/universal_sentence_encoder.py @@ -189,7 +189,7 @@ def loadSavedModel(folder, spark_session, loadsp=False): return UniversalSentenceEncoder(java_model=jModel) @staticmethod - def pretrained(name="tfhub_use", lang="en", remote_loc=None): + def pretrained(name="tfhub_use", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -208,4 +208,4 @@ def pretrained(name="tfhub_use", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(UniversalSentenceEncoder, name, lang, remote_loc) + return ResourceDownloader.downloadModel(UniversalSentenceEncoder, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/word2vec.py b/python/sparknlp/annotator/embeddings/word2vec.py index c9c9450f5ffb4e..5c93f5d08db46d 100755 --- a/python/sparknlp/annotator/embeddings/word2vec.py +++ b/python/sparknlp/annotator/embeddings/word2vec.py @@ -324,7 +324,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.embeddings.Word2VecModel", ja ) @staticmethod - def pretrained(name="word2vec_gigaword_300", lang="en", remote_loc=None): + def pretrained(name="word2vec_gigaword_300", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -343,7 +343,7 @@ def pretrained(name="word2vec_gigaword_300", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Word2VecModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(Word2VecModel, name, lang, remote_loc,engine) def getVectors(self): """ diff --git a/python/sparknlp/annotator/embeddings/word_embeddings.py b/python/sparknlp/annotator/embeddings/word_embeddings.py index 986c2822c0f87a..2c3f070ae84e08 100755 --- a/python/sparknlp/annotator/embeddings/word_embeddings.py +++ b/python/sparknlp/annotator/embeddings/word_embeddings.py @@ -348,7 +348,7 @@ def withCoverageColumn(dataset, embeddings_col, output_col='coverage'): return DataFrame(_EmbeddingsCoverageColumn(dataset, embeddings_col, output_col).apply(), dataset.sql_ctx) @staticmethod - def pretrained(name="glove_100d", lang="en", remote_loc=None): + def pretrained(name="glove_100d", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -367,7 +367,7 @@ def pretrained(name="glove_100d", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(WordEmbeddingsModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(WordEmbeddingsModel, name, lang, remote_loc,engine) @staticmethod def loadStorage(path, spark, storage_ref): diff --git a/python/sparknlp/annotator/embeddings/xlm_roberta_embeddings.py b/python/sparknlp/annotator/embeddings/xlm_roberta_embeddings.py index 76c33eb27f2eb0..1c141e0ab935f2 100755 --- a/python/sparknlp/annotator/embeddings/xlm_roberta_embeddings.py +++ b/python/sparknlp/annotator/embeddings/xlm_roberta_embeddings.py @@ -203,7 +203,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return XlmRoBertaEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="xlm_roberta_base", lang="xx", remote_loc=None): + def pretrained(name="xlm_roberta_base", lang="xx", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -222,4 +222,4 @@ def pretrained(name="xlm_roberta_base", lang="xx", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(XlmRoBertaEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/xlm_roberta_sentence_embeddings.py b/python/sparknlp/annotator/embeddings/xlm_roberta_sentence_embeddings.py index 89a5a7e86cc9b7..14a5ca4c5f44e1 100755 --- a/python/sparknlp/annotator/embeddings/xlm_roberta_sentence_embeddings.py +++ b/python/sparknlp/annotator/embeddings/xlm_roberta_sentence_embeddings.py @@ -172,7 +172,7 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaSentenceEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="sent_xlm_roberta_base", lang="xx", remote_loc=None): + def pretrained(name="sent_xlm_roberta_base", lang="xx", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -191,4 +191,4 @@ def pretrained(name="sent_xlm_roberta_base", lang="xx", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaSentenceEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(XlmRoBertaSentenceEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/embeddings/xlnet_embeddings.py b/python/sparknlp/annotator/embeddings/xlnet_embeddings.py index 44b406ceaa7b6b..4dc1d942f379a3 100755 --- a/python/sparknlp/annotator/embeddings/xlnet_embeddings.py +++ b/python/sparknlp/annotator/embeddings/xlnet_embeddings.py @@ -205,7 +205,7 @@ def loadSavedModel(folder, spark_session): return XlnetEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="xlnet_base_cased", lang="en", remote_loc=None): + def pretrained(name="xlnet_base_cased", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -224,4 +224,4 @@ def pretrained(name="xlnet_base_cased", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlnetEmbeddings, name, lang, remote_loc) + return ResourceDownloader.downloadModel(XlnetEmbeddings, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/ld_dl/language_detector_dl.py b/python/sparknlp/annotator/ld_dl/language_detector_dl.py index 462b321dd4c693..fda4bf822fe837 100755 --- a/python/sparknlp/annotator/ld_dl/language_detector_dl.py +++ b/python/sparknlp/annotator/ld_dl/language_detector_dl.py @@ -177,7 +177,7 @@ def setCoalesceSentences(self, value): return self._set(coalesceSentences=value) @staticmethod - def pretrained(name="ld_wiki_tatoeba_cnn_21", lang="xx", remote_loc=None): + def pretrained(name="ld_wiki_tatoeba_cnn_21", lang="xx", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -196,4 +196,4 @@ def pretrained(name="ld_wiki_tatoeba_cnn_21", lang="xx", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LanguageDetectorDL, name, lang, remote_loc) + return ResourceDownloader.downloadModel(LanguageDetectorDL, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/lemmatizer.py b/python/sparknlp/annotator/lemmatizer.py index 85d4dc9b3b7a97..771c207ed44ae9 100755 --- a/python/sparknlp/annotator/lemmatizer.py +++ b/python/sparknlp/annotator/lemmatizer.py @@ -228,7 +228,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.LemmatizerModel", ) @staticmethod - def pretrained(name="lemma_antbnc", lang="en", remote_loc=None): + def pretrained(name="lemma_antbnc", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -247,4 +247,4 @@ def pretrained(name="lemma_antbnc", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LemmatizerModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(LemmatizerModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/matcher/big_text_matcher.py b/python/sparknlp/annotator/matcher/big_text_matcher.py index 1718306eea8d17..825b080d665b48 100755 --- a/python/sparknlp/annotator/matcher/big_text_matcher.py +++ b/python/sparknlp/annotator/matcher/big_text_matcher.py @@ -234,7 +234,7 @@ def setCaseSensitive(self, v): return self._set(caseSensitive=v) @staticmethod - def pretrained(name, lang="en", remote_loc=None): + def pretrained(name, lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -253,7 +253,7 @@ def pretrained(name, lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(TextMatcherModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(TextMatcherModel, name, lang, remote_loc,engine) @staticmethod def loadStorage(path, spark, storage_ref): diff --git a/python/sparknlp/annotator/matcher/text_matcher.py b/python/sparknlp/annotator/matcher/text_matcher.py index 86a9dc26053e93..28fcc3edc155a6 100755 --- a/python/sparknlp/annotator/matcher/text_matcher.py +++ b/python/sparknlp/annotator/matcher/text_matcher.py @@ -268,7 +268,7 @@ def setBuildFromTokens(self, b): return self._set(buildFromTokens=b) @staticmethod - def pretrained(name, lang="en", remote_loc=None): + def pretrained(name, lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -287,4 +287,4 @@ def pretrained(name, lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(TextMatcherModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(TextMatcherModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/ner/ner_crf.py b/python/sparknlp/annotator/ner/ner_crf.py index 09b08222d7b8b1..8e2c58c257fc03 100755 --- a/python/sparknlp/annotator/ner/ner_crf.py +++ b/python/sparknlp/annotator/ner/ner_crf.py @@ -375,7 +375,7 @@ def setIncludeConfidence(self, b): return self._set(includeConfidence=b) @staticmethod - def pretrained(name="ner_crf", lang="en", remote_loc=None): + def pretrained(name="ner_crf", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -394,4 +394,4 @@ def pretrained(name="ner_crf", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(NerCrfModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(NerCrfModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/ner/ner_dl.py b/python/sparknlp/annotator/ner/ner_dl.py index 194468b16496c1..028c4d72fbec95 100755 --- a/python/sparknlp/annotator/ner/ner_dl.py +++ b/python/sparknlp/annotator/ner/ner_dl.py @@ -569,7 +569,7 @@ def setIncludeAllConfidenceScores(self, value): return self._set(includeAllConfidenceScores=value) @staticmethod - def pretrained(name="ner_dl", lang="en", remote_loc=None): + def pretrained(name="ner_dl", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -588,4 +588,4 @@ def pretrained(name="ner_dl", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(NerDLModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(NerDLModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/ner/zero_shot_ner_model.py b/python/sparknlp/annotator/ner/zero_shot_ner_model.py index 91412258299775..c1bf0294a17ead 100644 --- a/python/sparknlp/annotator/ner/zero_shot_ner_model.py +++ b/python/sparknlp/annotator/ner/zero_shot_ner_model.py @@ -161,9 +161,9 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.ner.dl.ZeroShotNer ) @staticmethod - def pretrained(name="zero_shot_ner_roberta", lang="en", remote_loc=None): + def pretrained(name="zero_shot_ner_roberta", lang="en", remote_loc=None,engine="onnx"): from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ZeroShotNerModel, name, lang, remote_loc, + return ResourceDownloader.downloadModel(ZeroShotNerModel, name, lang, remote_loc,engine, j_dwn='PythonResourceDownloader') @staticmethod diff --git a/python/sparknlp/annotator/pos/perceptron.py b/python/sparknlp/annotator/pos/perceptron.py index 1c461df7f9bd94..5b8aa1d34cca8f 100755 --- a/python/sparknlp/annotator/pos/perceptron.py +++ b/python/sparknlp/annotator/pos/perceptron.py @@ -241,7 +241,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.pos.perceptron.Per ) @staticmethod - def pretrained(name="pos_anc", lang="en", remote_loc=None): + def pretrained(name="pos_anc", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -260,4 +260,4 @@ def pretrained(name="pos_anc", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(PerceptronModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(PerceptronModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/sentence/sentence_detector_dl.py b/python/sparknlp/annotator/sentence/sentence_detector_dl.py index 6e7051e1a6dd59..e6437bea34645b 100755 --- a/python/sparknlp/annotator/sentence/sentence_detector_dl.py +++ b/python/sparknlp/annotator/sentence/sentence_detector_dl.py @@ -445,7 +445,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.sentence_detector_ ) @staticmethod - def pretrained(name="sentence_detector_dl", lang="en", remote_loc=None): + def pretrained(name="sentence_detector_dl", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -464,4 +464,4 @@ def pretrained(name="sentence_detector_dl", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SentenceDetectorDLModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(SentenceDetectorDLModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/sentiment/vivekn_sentiment.py b/python/sparknlp/annotator/sentiment/vivekn_sentiment.py index 53211165910d35..8e0452a3c7ee85 100755 --- a/python/sparknlp/annotator/sentiment/vivekn_sentiment.py +++ b/python/sparknlp/annotator/sentiment/vivekn_sentiment.py @@ -220,7 +220,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.sda.vivekn.ViveknS ) @staticmethod - def pretrained(name="sentiment_vivekn", lang="en", remote_loc=None): + def pretrained(name="sentiment_vivekn", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -239,4 +239,4 @@ def pretrained(name="sentiment_vivekn", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ViveknSentimentModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(ViveknSentimentModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/auto_gguf_model.py b/python/sparknlp/annotator/seq2seq/auto_gguf_model.py index 2495afdced304a..121db120a20cfe 100755 --- a/python/sparknlp/annotator/seq2seq/auto_gguf_model.py +++ b/python/sparknlp/annotator/seq2seq/auto_gguf_model.py @@ -277,7 +277,7 @@ def loadSavedModel(path, spark_session): return AutoGGUFModel(java_model=jModel) @staticmethod - def pretrained(name="Phi_4_mini_instruct_Q4_K_M_gguf", lang="en", remote_loc=None): + def pretrained(name="Phi_4_mini_instruct_Q4_K_M_gguf", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -296,7 +296,7 @@ def pretrained(name="Phi_4_mini_instruct_Q4_K_M_gguf", lang="en", remote_loc=Non The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AutoGGUFModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(AutoGGUFModel, name, lang, remote_loc,engine) def close(self): """Closes the llama.cpp model backend freeing resources. The model is reloaded when used again. diff --git a/python/sparknlp/annotator/seq2seq/auto_gguf_reranker.py b/python/sparknlp/annotator/seq2seq/auto_gguf_reranker.py index 075e45c67f5493..7fc1ebeaf62b97 100755 --- a/python/sparknlp/annotator/seq2seq/auto_gguf_reranker.py +++ b/python/sparknlp/annotator/seq2seq/auto_gguf_reranker.py @@ -307,7 +307,7 @@ def loadSavedModel(folder, spark_session): return AutoGGUFReranker(java_model=jModel) @staticmethod - def pretrained(name="bge_reranker_v2_m3_Q4_K_M", lang="en", remote_loc=None): + def pretrained(name="bge_reranker_v2_m3_Q4_K_M", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -326,7 +326,7 @@ def pretrained(name="bge_reranker_v2_m3_Q4_K_M", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AutoGGUFReranker, name, lang, remote_loc) + return ResourceDownloader.downloadModel(AutoGGUFReranker, name, lang, remote_loc,engine) def close(self): """Closes the llama.cpp model backend freeing resources. The model is reloaded when used again. diff --git a/python/sparknlp/annotator/seq2seq/auto_gguf_vision_model.py b/python/sparknlp/annotator/seq2seq/auto_gguf_vision_model.py index a632567b8b9dd1..d3d9f3277a4050 100755 --- a/python/sparknlp/annotator/seq2seq/auto_gguf_vision_model.py +++ b/python/sparknlp/annotator/seq2seq/auto_gguf_vision_model.py @@ -309,7 +309,7 @@ def loadSavedModel(modelPath, mmprojPath, spark_session): return AutoGGUFVisionModel(java_model=jModel) @staticmethod - def pretrained(name="Qwen2.5_VL_3B_Instruct_Q4_K_M_gguf", lang="en", remote_loc=None): + def pretrained(name="Qwen2.5_VL_3B_Instruct_Q4_K_M_gguf", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -328,7 +328,7 @@ def pretrained(name="Qwen2.5_VL_3B_Instruct_Q4_K_M_gguf", lang="en", remote_loc= The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AutoGGUFVisionModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(AutoGGUFVisionModel, name, lang, remote_loc,engine) def close(self): """Closes the llama.cpp model backend freeing resources. The model is reloaded when used again. diff --git a/python/sparknlp/annotator/seq2seq/bart_transformer.py b/python/sparknlp/annotator/seq2seq/bart_transformer.py index c9db8f039b43fe..0cdbc8aecd49ce 100755 --- a/python/sparknlp/annotator/seq2seq/bart_transformer.py +++ b/python/sparknlp/annotator/seq2seq/bart_transformer.py @@ -398,7 +398,7 @@ def loadSavedModel(folder, spark_session, use_cache=False): return BartTransformer(java_model=jModel) @staticmethod - def pretrained(name="distilbart_xsum_12_6", lang="en", remote_loc=None): + def pretrained(name="distilbart_xsum_12_6", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -417,4 +417,4 @@ def pretrained(name="distilbart_xsum_12_6", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BartTransformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(BartTransformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/cohere_transformer.py b/python/sparknlp/annotator/seq2seq/cohere_transformer.py index f72994860171a4..4b29b24057d4dd 100644 --- a/python/sparknlp/annotator/seq2seq/cohere_transformer.py +++ b/python/sparknlp/annotator/seq2seq/cohere_transformer.py @@ -335,7 +335,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return CoHereTransformer(java_model=jModel) @staticmethod - def pretrained(name="c4ai_command_r_v01_int4", lang="en", remote_loc=None): + def pretrained(name="c4ai_command_r_v01_int4", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -354,4 +354,4 @@ def pretrained(name="c4ai_command_r_v01_int4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CoHereTransformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(CoHereTransformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/cpm_transformer.py b/python/sparknlp/annotator/seq2seq/cpm_transformer.py index 7da0e216686fd6..c90114dc2d4b50 100644 --- a/python/sparknlp/annotator/seq2seq/cpm_transformer.py +++ b/python/sparknlp/annotator/seq2seq/cpm_transformer.py @@ -299,7 +299,7 @@ def loadSavedModel(folder, spark_session, use_openvino = False): return CPMTransformer(java_model=jModel) @staticmethod - def pretrained(name="mini_cpm_2b_8bit", lang="xx", remote_loc=None): + def pretrained(name="mini_cpm_2b_8bit", lang="xx", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -318,4 +318,4 @@ def pretrained(name="mini_cpm_2b_8bit", lang="xx", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CPMTransformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(CPMTransformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/gpt2_transformer.py b/python/sparknlp/annotator/seq2seq/gpt2_transformer.py index 346050c7ac0d4f..9adea769d91a98 100755 --- a/python/sparknlp/annotator/seq2seq/gpt2_transformer.py +++ b/python/sparknlp/annotator/seq2seq/gpt2_transformer.py @@ -341,7 +341,7 @@ def loadSavedModel(folder, spark_session): return GPT2Transformer(java_model=jModel) @staticmethod - def pretrained(name="gpt2", lang="en", remote_loc=None): + def pretrained(name="gpt2", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -360,4 +360,4 @@ def pretrained(name="gpt2", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(GPT2Transformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(GPT2Transformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/llama2_transformer.py b/python/sparknlp/annotator/seq2seq/llama2_transformer.py index d2d973ee4d7ced..dcaebd28658705 100644 --- a/python/sparknlp/annotator/seq2seq/llama2_transformer.py +++ b/python/sparknlp/annotator/seq2seq/llama2_transformer.py @@ -321,7 +321,7 @@ def loadSavedModel(folder, spark_session, use_openvino = False): return LLAMA2Transformer(java_model=jModel) @staticmethod - def pretrained(name="llama_2_7b_chat_hf_int4", lang="en", remote_loc=None): + def pretrained(name="llama_2_7b_chat_hf_int4", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -340,4 +340,4 @@ def pretrained(name="llama_2_7b_chat_hf_int4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LLAMA2Transformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(LLAMA2Transformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/llama3_transformer.py b/python/sparknlp/annotator/seq2seq/llama3_transformer.py index d4761bcadb56e0..9de976d5f45d1d 100644 --- a/python/sparknlp/annotator/seq2seq/llama3_transformer.py +++ b/python/sparknlp/annotator/seq2seq/llama3_transformer.py @@ -359,7 +359,7 @@ def loadSavedModel(folder, spark_session, use_openvino = False): return LLAMA3Transformer(java_model=jModel) @staticmethod - def pretrained(name="llama_3_7b_instruct_hf_int4", lang="en", remote_loc=None): + def pretrained(name="llama_3_7b_instruct_hf_int4", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -378,4 +378,4 @@ def pretrained(name="llama_3_7b_instruct_hf_int4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LLAMA3Transformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(LLAMA3Transformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/m2m100_transformer.py b/python/sparknlp/annotator/seq2seq/m2m100_transformer.py index cadf438cbd372b..3a3188474d095c 100644 --- a/python/sparknlp/annotator/seq2seq/m2m100_transformer.py +++ b/python/sparknlp/annotator/seq2seq/m2m100_transformer.py @@ -370,7 +370,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return M2M100Transformer(java_model=jModel) @staticmethod - def pretrained(name="m2m100_418M", lang="xx", remote_loc=None): + def pretrained(name="m2m100_418M", lang="xx", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -389,4 +389,4 @@ def pretrained(name="m2m100_418M", lang="xx", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(M2M100Transformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(M2M100Transformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/marian_transformer.py b/python/sparknlp/annotator/seq2seq/marian_transformer.py index a8586af9a4fd72..b76425d5c14ed0 100755 --- a/python/sparknlp/annotator/seq2seq/marian_transformer.py +++ b/python/sparknlp/annotator/seq2seq/marian_transformer.py @@ -352,7 +352,7 @@ def loadSavedModel(folder, spark_session): return MarianTransformer(java_model=jModel) @staticmethod - def pretrained(name="opus_mt_en_fr", lang="xx", remote_loc=None): + def pretrained(name="opus_mt_en_fr", lang="xx", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -371,4 +371,4 @@ def pretrained(name="opus_mt_en_fr", lang="xx", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MarianTransformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(MarianTransformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/mistral_transformer.py b/python/sparknlp/annotator/seq2seq/mistral_transformer.py index 1f810fe6305031..709ba534db8650 100644 --- a/python/sparknlp/annotator/seq2seq/mistral_transformer.py +++ b/python/sparknlp/annotator/seq2seq/mistral_transformer.py @@ -326,7 +326,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return MistralTransformer(java_model=jModel) @staticmethod - def pretrained(name="mistral_7b", lang="en", remote_loc=None): + def pretrained(name="mistral_7b", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -345,4 +345,4 @@ def pretrained(name="mistral_7b", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MistralTransformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(MistralTransformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/nllb_transformer.py b/python/sparknlp/annotator/seq2seq/nllb_transformer.py index e22efe1e3123ef..e4da9f1327028f 100644 --- a/python/sparknlp/annotator/seq2seq/nllb_transformer.py +++ b/python/sparknlp/annotator/seq2seq/nllb_transformer.py @@ -398,7 +398,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return NLLBTransformer(java_model=jModel) @staticmethod - def pretrained(name="nllb_distilled_600M_8int", lang="xx", remote_loc=None): + def pretrained(name="nllb_distilled_600M_8int", lang="xx", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -417,4 +417,4 @@ def pretrained(name="nllb_distilled_600M_8int", lang="xx", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(NLLBTransformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(NLLBTransformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/olmo_transformer.py b/python/sparknlp/annotator/seq2seq/olmo_transformer.py index eb1b63d71cdcf1..b8e2c54dd81c35 100644 --- a/python/sparknlp/annotator/seq2seq/olmo_transformer.py +++ b/python/sparknlp/annotator/seq2seq/olmo_transformer.py @@ -304,7 +304,7 @@ def loadSavedModel(folder, spark_session): return OLMoTransformer(java_model=jModel) @staticmethod - def pretrained(name="olmo_1b_int4", lang="en", remote_loc=None): + def pretrained(name="olmo_1b_int4", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -323,4 +323,4 @@ def pretrained(name="olmo_1b_int4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(OLMoTransformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(OLMoTransformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/phi2_transformer.py b/python/sparknlp/annotator/seq2seq/phi2_transformer.py index d2eaaad2b960e7..7e1a032ce0dabe 100644 --- a/python/sparknlp/annotator/seq2seq/phi2_transformer.py +++ b/python/sparknlp/annotator/seq2seq/phi2_transformer.py @@ -304,7 +304,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Phi2Transformer(java_model=jModel) @staticmethod - def pretrained(name="phi2", lang="en", remote_loc=None): + def pretrained(name="phi2", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -323,4 +323,4 @@ def pretrained(name="phi2", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Phi2Transformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(Phi2Transformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/phi3_transformer.py b/python/sparknlp/annotator/seq2seq/phi3_transformer.py index 98a28eeac47b96..530a9807fd15a8 100644 --- a/python/sparknlp/annotator/seq2seq/phi3_transformer.py +++ b/python/sparknlp/annotator/seq2seq/phi3_transformer.py @@ -308,7 +308,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Phi3Transformer(java_model=jModel) @staticmethod - def pretrained(name="phi_3_mini_128k_instruct", lang="en", remote_loc=None): + def pretrained(name="phi_3_mini_128k_instruct", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -327,4 +327,4 @@ def pretrained(name="phi_3_mini_128k_instruct", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Phi3Transformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(Phi3Transformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/phi4_transformer.py b/python/sparknlp/annotator/seq2seq/phi4_transformer.py index 8a55c191a30582..f863b76bc51579 100644 --- a/python/sparknlp/annotator/seq2seq/phi4_transformer.py +++ b/python/sparknlp/annotator/seq2seq/phi4_transformer.py @@ -56,7 +56,7 @@ class Phi4Transformer(AnnotatorModel, HasBatchedAnnotate, HasEngine): ... .setInputCols(["document"]) \ ... .setOutputCol("generation") - The default model is ``"phi-4"``, if no name is provided. For available pretrained models please see the `Models Hub `__. + The default model is ``"phi_4_mini_instruct_int8_openvino"``, if no name is provided. For available pretrained models please see the `Models Hub `__. Note ---- @@ -117,7 +117,7 @@ class Phi4Transformer(AnnotatorModel, HasBatchedAnnotate, HasEngine): >>> documentAssembler = DocumentAssembler() \ ... .setInputCol("text") \ ... .setOutputCol("documents") - >>> phi4 = Phi4Transformer.pretrained("phi-4") \ + >>> phi4 = Phi4Transformer.pretrained("phi_4_mini_instruct_int8_openvino") \ ... .setInputCols(["documents"]) \ ... .setMaxOutputLength(60) \ ... .setOutputCol("generation") @@ -365,13 +365,13 @@ def loadSavedModel(folder, spark_session, use_openvino = False): return Phi4Transformer(java_model=jModel) @staticmethod - def pretrained(name="phi-4", lang="en", remote_loc=None): + def pretrained(name="phi_4_mini_instruct_int8_openvino", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters ---------- name : str, optional - Name of the pretrained model, by default "phi-4" + Name of the pretrained model, by default "phi_4_mini_instruct_int8_openvino" lang : str, optional Language of the pretrained model, by default "en" remote_loc : str, optional @@ -384,4 +384,4 @@ def pretrained(name="phi-4", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Phi4Transformer, name, lang, remote_loc) \ No newline at end of file + return ResourceDownloader.downloadModel(Phi4Transformer, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/python/sparknlp/annotator/seq2seq/qwen_transformer.py b/python/sparknlp/annotator/seq2seq/qwen_transformer.py index 9f2257ad1edd98..ac73488661ef7c 100644 --- a/python/sparknlp/annotator/seq2seq/qwen_transformer.py +++ b/python/sparknlp/annotator/seq2seq/qwen_transformer.py @@ -318,7 +318,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return QwenTransformer(java_model=jModel) @staticmethod - def pretrained(name="qwen_7.5b_chat", lang="en", remote_loc=None): + def pretrained(name="qwen_7.5b_chat", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -337,4 +337,4 @@ def pretrained(name="qwen_7.5b_chat", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(QwenTransformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(QwenTransformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/starcoder_transformer.py b/python/sparknlp/annotator/seq2seq/starcoder_transformer.py index 3c87cb653682cc..b1850c28f2ca5b 100644 --- a/python/sparknlp/annotator/seq2seq/starcoder_transformer.py +++ b/python/sparknlp/annotator/seq2seq/starcoder_transformer.py @@ -313,7 +313,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return StarCoderTransformer(java_model=jModel) @staticmethod - def pretrained(name="starcoder", lang="en", remote_loc=None): + def pretrained(name="starcoder", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -332,4 +332,4 @@ def pretrained(name="starcoder", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(StarCoderTransformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(StarCoderTransformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/seq2seq/t5_transformer.py b/python/sparknlp/annotator/seq2seq/t5_transformer.py index 8191ba57fe5a62..c3ad0f8b40bfe1 100755 --- a/python/sparknlp/annotator/seq2seq/t5_transformer.py +++ b/python/sparknlp/annotator/seq2seq/t5_transformer.py @@ -403,7 +403,7 @@ def loadSavedModel(folder, spark_session): return T5Transformer(java_model=jModel) @staticmethod - def pretrained(name="t5_small", lang="en", remote_loc=None): + def pretrained(name="t5_small", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -422,4 +422,4 @@ def pretrained(name="t5_small", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(T5Transformer, name, lang, remote_loc) + return ResourceDownloader.downloadModel(T5Transformer, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/spell_check/context_spell_checker.py b/python/sparknlp/annotator/spell_check/context_spell_checker.py index a7ff483cab6145..039d0da86d6afd 100755 --- a/python/sparknlp/annotator/spell_check/context_spell_checker.py +++ b/python/sparknlp/annotator/spell_check/context_spell_checker.py @@ -889,7 +889,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.spell.context.Cont ) @staticmethod - def pretrained(name="spellcheck_dl", lang="en", remote_loc=None): + def pretrained(name="spellcheck_dl", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -908,4 +908,4 @@ def pretrained(name="spellcheck_dl", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ContextSpellCheckerModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(ContextSpellCheckerModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/spell_check/norvig_sweeting.py b/python/sparknlp/annotator/spell_check/norvig_sweeting.py index 74135d25fdfdeb..1918fee599e7d0 100755 --- a/python/sparknlp/annotator/spell_check/norvig_sweeting.py +++ b/python/sparknlp/annotator/spell_check/norvig_sweeting.py @@ -335,7 +335,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.spell.norvig.Norvi ) @staticmethod - def pretrained(name="spellcheck_norvig", lang="en", remote_loc=None): + def pretrained(name="spellcheck_norvig", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -354,5 +354,5 @@ def pretrained(name="spellcheck_norvig", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(NorvigSweetingModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(NorvigSweetingModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/spell_check/symmetric_delete.py b/python/sparknlp/annotator/spell_check/symmetric_delete.py index 34862efb9b35a3..ce216d0ff92883 100755 --- a/python/sparknlp/annotator/spell_check/symmetric_delete.py +++ b/python/sparknlp/annotator/spell_check/symmetric_delete.py @@ -276,7 +276,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.spell.symmetric.Sy ) @staticmethod - def pretrained(name="spellcheck_sd", lang="en", remote_loc=None): + def pretrained(name="spellcheck_sd", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -295,5 +295,5 @@ def pretrained(name="spellcheck_sd", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SymmetricDeleteModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(SymmetricDeleteModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/stop_words_cleaner.py b/python/sparknlp/annotator/stop_words_cleaner.py index 63536d394d4b18..6caf125de04bdc 100755 --- a/python/sparknlp/annotator/stop_words_cleaner.py +++ b/python/sparknlp/annotator/stop_words_cleaner.py @@ -168,7 +168,7 @@ def loadDefaultStopWords(language="english"): return list(stopWordsObj.loadDefaultStopWords(language)) @staticmethod - def pretrained(name="stopwords_en", lang="en", remote_loc=None): + def pretrained(name="stopwords_en", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -187,4 +187,4 @@ def pretrained(name="stopwords_en", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(StopWordsCleaner, name, lang, remote_loc) + return ResourceDownloader.downloadModel(StopWordsCleaner, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/token/tokenizer.py b/python/sparknlp/annotator/token/tokenizer.py index dd4113815d5a91..abfd76e482f806 100755 --- a/python/sparknlp/annotator/token/tokenizer.py +++ b/python/sparknlp/annotator/token/tokenizer.py @@ -539,7 +539,7 @@ def addSplitChars(self, value): return self._set(splitChars=split_chars) @staticmethod - def pretrained(name="token_rules", lang="en", remote_loc=None): + def pretrained(name="token_rules", lang="en", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -558,4 +558,4 @@ def pretrained(name="token_rules", lang="en", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(TokenizerModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(TokenizerModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/annotator/ws/word_segmenter.py b/python/sparknlp/annotator/ws/word_segmenter.py index 188de3bb21b388..2a6b8eb016bbe1 100755 --- a/python/sparknlp/annotator/ws/word_segmenter.py +++ b/python/sparknlp/annotator/ws/word_segmenter.py @@ -407,7 +407,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.ws.WordSegmenterMo ) @staticmethod - def pretrained(name="wordseg_pku", lang="zh", remote_loc=None): + def pretrained(name="wordseg_pku", lang="zh", remote_loc=None,engine="onnx"): """Downloads and loads a pretrained model. Parameters @@ -426,4 +426,4 @@ def pretrained(name="wordseg_pku", lang="zh", remote_loc=None): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(WordSegmenterModel, name, lang, remote_loc) + return ResourceDownloader.downloadModel(WordSegmenterModel, name, lang, remote_loc,engine) diff --git a/python/sparknlp/internal/__init__.py b/python/sparknlp/internal/__init__.py index 497cfa27964000..215af5c71ecece 100644 --- a/python/sparknlp/internal/__init__.py +++ b/python/sparknlp/internal/__init__.py @@ -713,13 +713,14 @@ def __init__(self, name, remote_loc="public/models", unzip=True): class _DownloadModel(ExtendedJavaWrapper): - def __init__(self, reader, name, language, remote_loc, validator): + def __init__(self, reader, name, language, remote_loc, engine, validator): super(_DownloadModel, self).__init__( "com.johnsnowlabs.nlp.pretrained." + validator + ".downloadModel", reader, name, language, remote_loc, + engine ) @@ -775,12 +776,14 @@ def __init__(self): class _GetResourceSize(ExtendedJavaWrapper): - def __init__(self, name, language, remote_loc): + def __init__(self, name, language, remote_loc, annotator, engine): super(_GetResourceSize, self).__init__( "com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader.getDownloadSize", name, language, remote_loc, + annotator, + engine ) diff --git a/python/sparknlp/pretrained/resource_downloader.py b/python/sparknlp/pretrained/resource_downloader.py index 00ffd0848a275b..2326609694bd0a 100644 --- a/python/sparknlp/pretrained/resource_downloader.py +++ b/python/sparknlp/pretrained/resource_downloader.py @@ -59,7 +59,7 @@ class ResourceDownloader(object): """ @staticmethod - def downloadModel(reader, name, language, remote_loc=None, j_dwn='PythonResourceDownloader'): + def downloadModel(reader, name, language, remote_loc=None, engine = "onnx", j_dwn='PythonResourceDownloader'): """Downloads and loads a model with the default downloader. Usually this method does not need to be called directly, as it is called by the `pretrained()` method of the annotator. @@ -83,7 +83,7 @@ def downloadModel(reader, name, language, remote_loc=None, j_dwn='PythonResource Loaded pretrained annotator/pipeline """ print(name + " download started this may take some time.") - file_size = _internal._GetResourceSize(name, language, remote_loc).apply() + file_size = _internal._GetResourceSize(name, language, remote_loc, reader.name, engine).apply() if file_size == "-1": print("Can not find the model to download please check the name!") else: @@ -92,7 +92,7 @@ def downloadModel(reader, name, language, remote_loc=None, j_dwn='PythonResource t1 = threading.Thread(target=printProgress, args=(lambda: stop_threads,)) t1.start() try: - j_obj = _internal._DownloadModel(reader.name, name, language, remote_loc, j_dwn).apply() + j_obj = _internal._DownloadModel(reader.name, name, language, remote_loc, engine, j_dwn).apply() except Py4JJavaError as e: sys.stdout.write("\n" + str(e)) raise e diff --git a/python/test/annotator/classifier_dl/distilbert_for_token_classification_test.py b/python/test/annotator/classifier_dl/distilbert_for_token_classification_test.py index 10a42ed64f8c26..b196fc68e5e433 100644 --- a/python/test/annotator/classifier_dl/distilbert_for_token_classification_test.py +++ b/python/test/annotator/classifier_dl/distilbert_for_token_classification_test.py @@ -28,8 +28,8 @@ def setUp(self): self.data = SparkContextForTest.spark.read.option("header", "true") \ .csv(path="file:///" + os.getcwd() + "/../src/test/resources/embeddings/sentence_embeddings.csv") - self.tested_annotator = DistilBertForTokenClassification.pretrained() \ - .setInputCols(["document", "token"]) \ + self.tested_annotator = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_keyphrase_extraction_inspec",engine="openvino") \ + .setInputCols(["document", "token"]) \ .setOutputCol("ner") def test_run(self): @@ -50,5 +50,5 @@ def test_run(self): model = pipeline.fit(self.data) model.transform(self.data).show() - print(self.classifier.getClasses()) - print(self.classifier.getBatchSize()) + #print(self.classifier.getClasses()) + #print(self.classifier.getBatchSize()) diff --git a/src/main/scala/com/johnsnowlabs/nlp/HasPretrained.scala b/src/main/scala/com/johnsnowlabs/nlp/HasPretrained.scala index 41e2315783d4bc..db7760e4cfbe42 100644 --- a/src/main/scala/com/johnsnowlabs/nlp/HasPretrained.scala +++ b/src/main/scala/com/johnsnowlabs/nlp/HasPretrained.scala @@ -29,6 +29,8 @@ trait HasPretrained[M <: PipelineStage] { val defaultLang: String = "en" + val defaultPreferredEngine: String = "onnx" + lazy val defaultLoc: String = ResourceDownloader.publicLoc implicit private val companion: DefaultParamsReadable[M] = @@ -38,17 +40,35 @@ trait HasPretrained[M <: PipelineStage] { s"${this.getClass.getName} does not have a default pretrained model. Please provide a model name." /** Java default argument interoperability */ - def pretrained(name: String, lang: String, remoteLoc: String): M = { + def pretrained( + name: String, + lang: String, + remoteLoc: String, + preferredEngine: String = "onnx"): M = { if (Option(name).isEmpty) throw new NotImplementedError(errorMsg) - ResourceDownloader.downloadModel(companion, name, Option(lang), remoteLoc) + ResourceDownloader.downloadModel(companion, name, Option(lang), remoteLoc, preferredEngine) } - def pretrained(name: String, lang: String): M = pretrained(name, lang, defaultLoc) + def pretrained(name: String, lang: String): M = + pretrained(name, lang, defaultLoc, defaultPreferredEngine) - def pretrained(name: String): M = pretrained(name, defaultLang, defaultLoc) + def pretrained(name: String): M = + pretrained(name, defaultLang, defaultLoc, defaultPreferredEngine) def pretrained(): M = - pretrained(defaultModelName.getOrElse(throw new Exception(errorMsg)), defaultLang, defaultLoc) + pretrained( + defaultModelName.getOrElse(throw new Exception(errorMsg)), + defaultLang, + defaultLoc, + defaultPreferredEngine) + + def pretrained(name: String, lang: String, remoteLoc: String): M = + pretrained(name, lang, remoteLoc, defaultPreferredEngine) + def pretrainedEngine(name: String, preferredEngine: String): M = + pretrained(name, defaultLang, defaultLoc, preferredEngine) + + def pretrainedEngine(name: String, lang: String, preferredEngine: String): M = + pretrained(name, lang, defaultLoc, preferredEngine) } diff --git a/src/main/scala/com/johnsnowlabs/nlp/annotators/seq2seq/Phi4Transformer.scala b/src/main/scala/com/johnsnowlabs/nlp/annotators/seq2seq/Phi4Transformer.scala index c4f9cb9df62d44..e55cd2f851eef4 100644 --- a/src/main/scala/com/johnsnowlabs/nlp/annotators/seq2seq/Phi4Transformer.scala +++ b/src/main/scala/com/johnsnowlabs/nlp/annotators/seq2seq/Phi4Transformer.scala @@ -250,12 +250,13 @@ class Phi4Transformer(override val uid: String) trait ReadablePretrainedPhi4TransformerModel extends ParamsAndFeaturesReadable[Phi4Transformer] with HasPretrained[Phi4Transformer] { - override val defaultModelName: Some[String] = Some("phi-4") + override val defaultModelName: Some[String] = Some("phi_4_mini_instruct_int8_openvino") override def pretrained(): Phi4Transformer = super.pretrained() override def pretrained(name: String): Phi4Transformer = super.pretrained(name) override def pretrained(name: String, lang: String): Phi4Transformer = super.pretrained(name, lang) + override def pretrained(name: String, lang: String, remoteLoc: String): Phi4Transformer = super.pretrained(name, lang, remoteLoc) } diff --git a/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceDownloader.scala b/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceDownloader.scala index c0e54ca1769634..3bbc562fb953dd 100644 --- a/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceDownloader.scala +++ b/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceDownloader.scala @@ -496,8 +496,18 @@ object ResourceDownloader { reader: DefaultParamsReadable[TModel], name: String, language: Option[String] = None, - folder: String = publicLoc): TModel = { - downloadModel(reader, ResourceRequest(name, language, folder)) + folder: String = publicLoc, + preferredEngine: String = "onnx"): TModel = { + + val annotator = reader.getClass.getSimpleName.replace("$", "") + downloadModel( + reader, + ResourceRequest( + name, + language, + folder, + annotator = Some(annotator), + engine = Some(preferredEngine))) } def downloadModel[TModel <: PipelineStage]( @@ -517,7 +527,7 @@ object ResourceDownloader { name: String, language: Option[String] = None, folder: String = publicLoc): PipelineModel = { - downloadPipeline(ResourceRequest(name, language, folder)) + downloadPipeline(ResourceRequest(name, language, folder, annotator = Some("PipelineModel"))) } def downloadPipeline(request: ResourceRequest): PipelineModel = { @@ -575,7 +585,9 @@ case class ResourceRequest( language: Option[String] = None, folder: String = ResourceDownloader.publicLoc, libVersion: Version = ResourceDownloader.libVersion, - sparkVersion: Version = ResourceDownloader.sparkVersion) + sparkVersion: Version = ResourceDownloader.sparkVersion, + annotator: Option[String] = None, + engine: Option[String] = None) /* convenience accessor for Py4J calls */ object PythonResourceDownloader { @@ -632,7 +644,6 @@ object PythonResourceDownloader { "XlnetForTokenClassification" -> XlnetForTokenClassification, "AlbertForSequenceClassification" -> AlbertForSequenceClassification, "BertForSequenceClassification" -> BertForSequenceClassification, - "DeBertaForSequenceClassification" -> DeBertaForSequenceClassification, "DistilBertForSequenceClassification" -> DistilBertForSequenceClassification, "LongformerForSequenceClassification" -> LongformerForSequenceClassification, "RoBertaForSequenceClassification" -> RoBertaForSequenceClassification, @@ -644,7 +655,6 @@ object PythonResourceDownloader { "Word2VecModel" -> Word2VecModel, "DeBertaEmbeddings" -> DeBertaEmbeddings, "DeBertaForSequenceClassification" -> DeBertaForSequenceClassification, - "DeBertaForTokenClassification" -> DeBertaForTokenClassification, "CamemBertEmbeddings" -> CamemBertEmbeddings, "AlbertForQuestionAnswering" -> AlbertForQuestionAnswering, "BertForQuestionAnswering" -> BertForQuestionAnswering, @@ -722,7 +732,8 @@ object PythonResourceDownloader { readerStr: String, name: String, language: String = null, - remoteLoc: String = null): PipelineStage = { + remoteLoc: String = null, + preferredEngine: String): PipelineStage = { val reader = keyToReader.getOrElse( if (typeMapper.contains(readerStr)) typeMapper(readerStr) else readerStr, @@ -734,7 +745,8 @@ object PythonResourceDownloader { reader.asInstanceOf[DefaultParamsReadable[PipelineStage]], name, Option(language), - correctedFolder) + correctedFolder, + preferredEngine) // Cast the model to the required type. This has to be done for each entry in the typeMapper map if (typeMapper.contains(readerStr) && readerStr == "ZeroShotNerModel") @@ -800,8 +812,19 @@ object PythonResourceDownloader { ResourceDownloader.listAvailableAnnotators().mkString("\n") } - def getDownloadSize(name: String, language: String = "en", remoteLoc: String = null): String = { + def getDownloadSize( + name: String, + language: String = "en", + remoteLoc: String = null, + annotator: String, + engine: String): String = { val correctedFolder = Option(remoteLoc).getOrElse(ResourceDownloader.publicLoc) - ResourceDownloader.getDownloadSize(ResourceRequest(name, Option(language), correctedFolder)) + ResourceDownloader.getDownloadSize( + ResourceRequest( + name, + Option(language), + correctedFolder, + annotator = Some(annotator), + engine = Some(engine))) } } diff --git a/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceMetadata.scala b/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceMetadata.scala index 992708e86c0992..50dd946fe0fd84 100644 --- a/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceMetadata.scala +++ b/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceMetadata.scala @@ -109,15 +109,44 @@ object ResourceMetadata { candidates: List[ResourceMetadata], request: ResourceRequest): Option[ResourceMetadata] = { - val compatibleCandidates = candidates - .filter(item => - item.readyToUse && item.libVersion.isDefined && item.sparkVersion.isDefined - && item.name == request.name - && (request.language.isEmpty || item.language.isEmpty || request.language.get == item.language.get) - && Version.isCompatible(request.libVersion, item.libVersion) - && Version.isCompatible(request.sparkVersion, item.sparkVersion)) - - val sortedResult = compatibleCandidates.sorted + val compatibleCandidates = candidates.filter(item => + item.readyToUse && + item.libVersion.isDefined && + item.sparkVersion.isDefined && + item.name == request.name && + (request.annotator.isEmpty || item.annotator.isEmpty || + request.annotator.get.equalsIgnoreCase(item.annotator.get)) && + (request.language.isEmpty || item.language.isEmpty || + request.language.get == item.language.get) && + Version.isCompatible(request.libVersion, item.libVersion) && + Version.isCompatible(request.sparkVersion, item.sparkVersion)) + + val defaultPriority = Seq("onnx", "tensorflow", "openvino", "unk") + + val finalPriority = request.engine match { + case Some(pref) => + val p = pref.toLowerCase // incase user types ONNX instead of onnx + (Seq(p) ++ defaultPriority.filterNot(_ == p)).distinct + case None => + defaultPriority + } + + def enginePriority(engineOpt: Option[String]): Int = { + val engine = engineOpt.map(_.toLowerCase).getOrElse("unk") + finalPriority.indexOf(engine) match { + case -1 => + finalPriority.length // unknown engine → lowest priority ( therefore highest numerical value ) + case idx => idx + } + } + + // Engine → Spark → Lib → Time + val sortedResult = compatibleCandidates.sortWith { (a, b) => + val engineComp = enginePriority(a.engine) compare enginePriority(b.engine) + if (engineComp != 0) engineComp > 0 + else a < b // fallback to old logic + } + sortedResult.lastOption } From 1e1481d5230fef9ccc4fab61368e2a29042f4f55 Mon Sep 17 00:00:00 2001 From: ahmedlone127 Date: Fri, 7 Nov 2025 12:13:56 +0500 Subject: [PATCH 2/8] updating default model name --- python/sparknlp/annotator/cv/florence2_transformer.py | 2 +- .../johnsnowlabs/nlp/annotators/cv/Florence2Transformer.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/sparknlp/annotator/cv/florence2_transformer.py b/python/sparknlp/annotator/cv/florence2_transformer.py index e79882eef1131d..f0dcc0d718dba7 100644 --- a/python/sparknlp/annotator/cv/florence2_transformer.py +++ b/python/sparknlp/annotator/cv/florence2_transformer.py @@ -174,7 +174,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Florence2Transformer(java_model=jModel) @staticmethod - def pretrained(name="florence2_base_ft_int4", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="florence_2_base_ft_int4", lang="en", remote_loc=None,engine ="onnx"): """Downloads and loads a pretrained model.""" from sparknlp.pretrained import ResourceDownloader return ResourceDownloader.downloadModel(Florence2Transformer, name, lang, remote_loc,engine) \ No newline at end of file diff --git a/src/main/scala/com/johnsnowlabs/nlp/annotators/cv/Florence2Transformer.scala b/src/main/scala/com/johnsnowlabs/nlp/annotators/cv/Florence2Transformer.scala index a570d2cbdf07c4..4b1b883cd938b1 100644 --- a/src/main/scala/com/johnsnowlabs/nlp/annotators/cv/Florence2Transformer.scala +++ b/src/main/scala/com/johnsnowlabs/nlp/annotators/cv/Florence2Transformer.scala @@ -330,7 +330,7 @@ class Florence2Transformer(override val uid: String) trait ReadablePretrainedFlorence2TransformerModel extends ParamsAndFeaturesReadable[Florence2Transformer] with HasPretrained[Florence2Transformer] { - override val defaultModelName: Some[String] = Some("florence2_base_ft_int4") + override val defaultModelName: Some[String] = Some("florence_2_base_ft_int4") /** Java compliant-overrides */ override def pretrained(): Florence2Transformer = super.pretrained() From 2d5bfa1b2039f0b9cbd28c57b7deab856ebdda3a Mon Sep 17 00:00:00 2001 From: Abdullah mubeen Date: Thu, 13 Nov 2025 21:19:29 +0500 Subject: [PATCH 3/8] Specify model name in E5VEmbeddings test default model no longer exists --- .../com/johnsnowlabs/nlp/embeddings/E5VEmbeddingsTestSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddingsTestSpec.scala b/src/test/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddingsTestSpec.scala index 9bc4d00f9b7eb2..ded74789da034a 100644 --- a/src/test/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddingsTestSpec.scala +++ b/src/test/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddingsTestSpec.scala @@ -69,7 +69,7 @@ class E5VEmbeddingsTestSpec extends AnyFlatSpec { .setOutputCol("image_assembler") val loadModel = E5VEmbeddings - .pretrained() + .pretrained("e5v_int4", "en") .setInputCols("image_assembler") .setOutputCol("e5v") From 45e581ead3f8cdfd80dc3177b66ab8dbd0dc6b2a Mon Sep 17 00:00:00 2001 From: Abdullah mubeen Date: Thu, 13 Nov 2025 21:41:58 +0500 Subject: [PATCH 4/8] Revert "Specify model name in E5VEmbeddings test" This reverts commit 66d19f63119328f7a9e55c2d79e4ef011cc7f614. --- .../com/johnsnowlabs/nlp/embeddings/E5VEmbeddingsTestSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddingsTestSpec.scala b/src/test/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddingsTestSpec.scala index ded74789da034a..9bc4d00f9b7eb2 100644 --- a/src/test/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddingsTestSpec.scala +++ b/src/test/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddingsTestSpec.scala @@ -69,7 +69,7 @@ class E5VEmbeddingsTestSpec extends AnyFlatSpec { .setOutputCol("image_assembler") val loadModel = E5VEmbeddings - .pretrained("e5v_int4", "en") + .pretrained() .setInputCols("image_assembler") .setOutputCol("e5v") From 906c8b9abb1c7c1ad099aba2e67b6c472bc75cc2 Mon Sep 17 00:00:00 2001 From: Abdullah mubeen Date: Fri, 14 Nov 2025 00:16:19 +0500 Subject: [PATCH 5/8] Update default model for E5VEmbeddings --- .../scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddings.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddings.scala b/src/main/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddings.scala index 657d012c04734e..aa05f43daa7d36 100644 --- a/src/main/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddings.scala +++ b/src/main/scala/com/johnsnowlabs/nlp/embeddings/E5VEmbeddings.scala @@ -376,7 +376,7 @@ trait ReadablePretrainedE5VEmbeddings extends ParamsAndFeaturesReadable[E5VEmbeddings] with HasPretrained[E5VEmbeddings] { - override val defaultModelName: Some[String] = Some("e5v_1_5_7b_int4") + override val defaultModelName: Some[String] = Some("e5v_int4") /** Java compliant-overrides */ override def pretrained(): E5VEmbeddings = super.pretrained() From bd0e6033841cdb21b11dc58983265d819a2c6fd3 Mon Sep 17 00:00:00 2001 From: ahmedlone127 Date: Thu, 27 Nov 2025 17:56:06 +0500 Subject: [PATCH 6/8] SPARKNLP-1303 adding python test for testing perfered engine behaviour --- .../test/annotator/embeddings/camembert_embeddings_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/test/annotator/embeddings/camembert_embeddings_test.py b/python/test/annotator/embeddings/camembert_embeddings_test.py index a9795b7c68c550..94576cdd546e9a 100644 --- a/python/test/annotator/embeddings/camembert_embeddings_test.py +++ b/python/test/annotator/embeddings/camembert_embeddings_test.py @@ -48,3 +48,9 @@ def test_run(self): model = pipeline.fit(self.data) model.transform(self.data).show() + + def test_perferred_engine(self): + model_onnx = CamemBertEmbeddings.pretrained("camembert_base","fr",engine= "onnx") + model_tensorflow = CamemBertEmbeddings.pretrained("camembert_base","fr",engine= "tensorflow") + self.assertEqual(model_onnx.getEngine(), "onnx") + self.assertEqual(model_tensorflow.getEngine(), "tensorflow") From 68b5249131f0dba80ca845ecbd88287a901375c1 Mon Sep 17 00:00:00 2001 From: ahmedlone127 Date: Thu, 27 Nov 2025 23:23:26 +0500 Subject: [PATCH 7/8] SPARKNLP-1303 updating all python side pretrained functions and removing engine parameter --- .../annotator/audio/hubert_for_ctc.py | 4 ++-- .../annotator/audio/wav2vec2_for_ctc.py | 4 ++-- .../annotator/audio/whisper_for_ctc.py | 4 ++-- .../albert_for_multiple_choice.py | 4 ++-- .../albert_for_question_answering.py | 4 ++-- .../albert_for_sequence_classification.py | 4 ++-- .../albert_for_token_classification.py | 4 ++-- .../albert_for_zero_shot_classification.py | 4 ++-- .../bart_for_zero_shot_classification.py | 4 ++-- .../classifier_dl/bert_for_multiple_choice.py | 4 ++-- .../bert_for_question_answering.py | 4 ++-- .../bert_for_sequence_classification.py | 4 ++-- .../bert_for_token_classification.py | 4 ++-- .../bert_for_zero_shot_classification.py | 4 ++-- .../camembert_for_question_answering.py | 4 ++-- .../camembert_for_sequence_classification.py | 4 ++-- .../camembert_for_token_classification.py | 4 ++-- .../camembert_for_zero_shot_classification.py | 4 ++-- .../annotator/classifier_dl/classifier_dl.py | 4 ++-- .../deberta_for_question_answering.py | 4 ++-- .../deberta_for_sequence_classification.py | 4 ++-- .../deberta_for_token_classification.py | 4 ++-- .../deberta_for_zero_shot_classification.py | 4 ++-- .../distil_bert_for_question_answering.py | 4 ++-- ...distil_bert_for_sequence_classification.py | 4 ++-- .../distil_bert_for_token_classification.py | 4 ++-- ...istil_bert_for_zero_shot_classification.py | 4 ++-- .../distilbert_for_multiple_choice.py | 4 ++-- .../longformer_for_question_answering.py | 4 ++-- .../longformer_for_sequence_classification.py | 4 ++-- .../longformer_for_token_classification.py | 4 ++-- .../mpnet_for_question_answering.py | 4 ++-- .../mpnet_for_sequence_classification.py | 4 ++-- .../mpnet_for_token_classification.py | 4 ++-- .../classifier_dl/multi_classifier_dl.py | 4 ++-- .../roberta_for_multiple_choice.py | 4 ++-- .../roberta_for_question_answering.py | 4 ++-- .../roberta_for_sequence_classification.py | 4 ++-- .../roberta_for_token_classification.py | 4 ++-- .../roberta_for_zero_shot_classification.py | 4 ++-- .../annotator/classifier_dl/sentiment_dl.py | 4 ++-- .../tapas_for_question_answering.py | 4 ++-- .../xlm_roberta_for_multiple_choice.py | 4 ++-- .../xlm_roberta_for_question_answering.py | 4 ++-- ...xlm_roberta_for_sequence_classification.py | 4 ++-- .../xlm_roberta_for_token_classification.py | 4 ++-- ...lm_roberta_for_zero_shot_classification.py | 4 ++-- .../xlnet_for_sequence_classification.py | 4 ++-- .../xlnet_for_token_classification.py | 4 ++-- .../annotator/coref/spanbert_coref.py | 4 ++-- .../cv/blip_for_question_answering.py | 4 ++-- .../cv/clip_for_zero_shot_classification.py | 4 ++-- .../cv/convnext_for_image_classification.py | 4 ++-- .../annotator/cv/florence2_transformer.py | 4 ++-- .../annotator/cv/gemma3_for_multimodal.py | 4 ++-- .../annotator/cv/internvl_for_multimodal.py | 4 ++-- .../annotator/cv/janus_for_multimodal.py | 4 ++-- .../annotator/cv/llava_for_multimodal.py | 4 ++-- .../annotator/cv/mllama_for_multimodal.py | 4 ++-- .../annotator/cv/paligemma_for_multimodal.py | 4 ++-- .../cv/phi3_vision_for_multimodal.py | 4 ++-- .../annotator/cv/qwen2vl_transformer.py | 4 ++-- .../annotator/cv/smolvlm_transformer.py | 4 ++-- .../cv/swin_for_image_classification.py | 4 ++-- ...on_encoder_decoder_for_image_captioning.py | 4 ++-- .../cv/vit_for_image_classification.py | 4 ++-- .../annotator/dependency/dependency_parser.py | 4 ++-- .../dependency/typed_dependency_parser.py | 4 ++-- .../annotator/embeddings/albert_embeddings.py | 4 ++-- .../embeddings/auto_gguf_embeddings.py | 4 ++-- .../annotator/embeddings/bert_embeddings.py | 4 ++-- .../embeddings/bert_sentence_embeddings.py | 4 ++-- .../annotator/embeddings/bge_embeddings.py | 4 ++-- .../embeddings/camembert_embeddings.py | 4 ++-- .../embeddings/deberta_embeddings.py | 4 ++-- .../embeddings/distil_bert_embeddings.py | 4 ++-- .../sparknlp/annotator/embeddings/doc2vec.py | 4 ++-- .../annotator/embeddings/e5_embeddings.py | 4 ++-- .../annotator/embeddings/e5v_embeddings.py | 4 ++-- .../annotator/embeddings/elmo_embeddings.py | 4 ++-- .../embeddings/instructor_embeddings.py | 4 ++-- .../embeddings/longformer_embeddings.py | 4 ++-- .../annotator/embeddings/minilm_embeddings.py | 4 ++-- .../annotator/embeddings/mpnet_embeddings.py | 4 ++-- .../annotator/embeddings/mxbai_embeddings.py | 4 ++-- .../annotator/embeddings/nomic_embeddings.py | 4 ++-- .../embeddings/roberta_embeddings.py | 4 ++-- .../embeddings/roberta_sentence_embeddings.py | 4 ++-- .../embeddings/snowflake_embeddings.py | 4 ++-- .../annotator/embeddings/uae_embeddings.py | 4 ++-- .../embeddings/universal_sentence_encoder.py | 4 ++-- .../sparknlp/annotator/embeddings/word2vec.py | 4 ++-- .../annotator/embeddings/word_embeddings.py | 4 ++-- .../embeddings/xlm_roberta_embeddings.py | 4 ++-- .../xlm_roberta_sentence_embeddings.py | 4 ++-- .../annotator/embeddings/xlnet_embeddings.py | 4 ++-- .../annotator/ld_dl/language_detector_dl.py | 4 ++-- python/sparknlp/annotator/lemmatizer.py | 4 ++-- .../annotator/matcher/big_text_matcher.py | 4 ++-- .../annotator/matcher/text_matcher.py | 4 ++-- python/sparknlp/annotator/ner/ner_crf.py | 4 ++-- python/sparknlp/annotator/ner/ner_dl.py | 4 ++-- .../annotator/ner/zero_shot_ner_model.py | 4 ++-- python/sparknlp/annotator/pos/perceptron.py | 4 ++-- .../sentence/sentence_detector_dl.py | 4 ++-- .../annotator/sentiment/vivekn_sentiment.py | 4 ++-- .../annotator/seq2seq/auto_gguf_model.py | 4 ++-- .../annotator/seq2seq/auto_gguf_reranker.py | 4 ++-- .../seq2seq/auto_gguf_vision_model.py | 4 ++-- .../annotator/seq2seq/bart_transformer.py | 4 ++-- .../annotator/seq2seq/cohere_transformer.py | 4 ++-- .../annotator/seq2seq/cpm_transformer.py | 4 ++-- .../annotator/seq2seq/gpt2_transformer.py | 4 ++-- .../annotator/seq2seq/llama2_transformer.py | 4 ++-- .../annotator/seq2seq/llama3_transformer.py | 4 ++-- .../annotator/seq2seq/m2m100_transformer.py | 4 ++-- .../annotator/seq2seq/marian_transformer.py | 4 ++-- .../annotator/seq2seq/mistral_transformer.py | 4 ++-- .../annotator/seq2seq/nllb_transformer.py | 4 ++-- .../annotator/seq2seq/olmo_transformer.py | 4 ++-- .../annotator/seq2seq/phi2_transformer.py | 4 ++-- .../annotator/seq2seq/phi3_transformer.py | 4 ++-- .../annotator/seq2seq/phi4_transformer.py | 4 ++-- .../annotator/seq2seq/qwen_transformer.py | 4 ++-- .../seq2seq/starcoder_transformer.py | 4 ++-- .../annotator/seq2seq/t5_transformer.py | 4 ++-- .../spell_check/context_spell_checker.py | 4 ++-- .../annotator/spell_check/norvig_sweeting.py | 4 ++-- .../annotator/spell_check/symmetric_delete.py | 4 ++-- .../sparknlp/annotator/stop_words_cleaner.py | 4 ++-- python/sparknlp/annotator/token/tokenizer.py | 4 ++-- .../sparknlp/annotator/ws/word_segmenter.py | 4 ++-- python/sparknlp/common/properties.py | 23 +++++++++++++++++++ ...istilbert_for_token_classification_test.py | 8 +++---- .../embeddings/camembert_embeddings_test.py | 4 ++-- 135 files changed, 293 insertions(+), 270 deletions(-) diff --git a/python/sparknlp/annotator/audio/hubert_for_ctc.py b/python/sparknlp/annotator/audio/hubert_for_ctc.py index 65e4c03d5ec2ac..0f7549e712ae4e 100644 --- a/python/sparknlp/annotator/audio/hubert_for_ctc.py +++ b/python/sparknlp/annotator/audio/hubert_for_ctc.py @@ -165,7 +165,7 @@ def loadSavedModel(folder, spark_session): return HubertForCTC(java_model=jModel) @staticmethod - def pretrained(name="asr_hubert_large_ls960", lang="en", remote_loc=None, engine ="onnx"): + def pretrained(name="asr_hubert_large_ls960", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -185,4 +185,4 @@ def pretrained(name="asr_hubert_large_ls960", lang="en", remote_loc=None, engine The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(HubertForCTC, name, lang, remote_loc, engine) + return ResourceDownloader.downloadModel(HubertForCTC, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/audio/wav2vec2_for_ctc.py b/python/sparknlp/annotator/audio/wav2vec2_for_ctc.py index f4f3fe9f23bd94..c1937df094e5c8 100644 --- a/python/sparknlp/annotator/audio/wav2vec2_for_ctc.py +++ b/python/sparknlp/annotator/audio/wav2vec2_for_ctc.py @@ -138,7 +138,7 @@ def loadSavedModel(folder, spark_session): return Wav2Vec2ForCTC(java_model=jModel) @staticmethod - def pretrained(name="asr_wav2vec2_base_960h", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="asr_wav2vec2_base_960h", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -158,4 +158,4 @@ def pretrained(name="asr_wav2vec2_base_960h", lang="en", remote_loc=None,engine The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Wav2Vec2ForCTC, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(Wav2Vec2ForCTC, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/audio/whisper_for_ctc.py b/python/sparknlp/annotator/audio/whisper_for_ctc.py index 446d3116ae138a..343bf4749a3714 100644 --- a/python/sparknlp/annotator/audio/whisper_for_ctc.py +++ b/python/sparknlp/annotator/audio/whisper_for_ctc.py @@ -228,7 +228,7 @@ def loadSavedModel(folder, spark_session): return WhisperForCTC(java_model=jModel) @staticmethod - def pretrained(name="asr_whisper_tiny_opt", lang="xx", remote_loc=None,engine ="onnx"): + def pretrained(name="asr_whisper_tiny_opt", lang="xx", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -248,4 +248,4 @@ def pretrained(name="asr_whisper_tiny_opt", lang="xx", remote_loc=None,engine =" The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(WhisperForCTC, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(WhisperForCTC, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/albert_for_multiple_choice.py b/python/sparknlp/annotator/classifier_dl/albert_for_multiple_choice.py index 25d5b6391c57fd..7dc610b256f687 100644 --- a/python/sparknlp/annotator/classifier_dl/albert_for_multiple_choice.py +++ b/python/sparknlp/annotator/classifier_dl/albert_for_multiple_choice.py @@ -138,7 +138,7 @@ def loadSavedModel(folder, spark_session): return AlbertForMultipleChoice(java_model=jModel) @staticmethod - def pretrained(name="albert_base_uncased_multiple_choice", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="albert_base_uncased_multiple_choice", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -158,4 +158,4 @@ def pretrained(name="albert_base_uncased_multiple_choice", lang="en", remote_loc The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertForMultipleChoice, name, lang, remote_loc,engine ) \ No newline at end of file + return ResourceDownloader.downloadModel(AlbertForMultipleChoice, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/albert_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/albert_for_question_answering.py index b771f754e26d11..113cdbfb801184 100755 --- a/python/sparknlp/annotator/classifier_dl/albert_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/albert_for_question_answering.py @@ -149,7 +149,7 @@ def loadSavedModel(folder, spark_session): return AlbertForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="albert_base_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="albert_base_qa_squad2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -169,4 +169,4 @@ def pretrained(name="albert_base_qa_squad2", lang="en", remote_loc=None,engine = The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertForQuestionAnswering, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(AlbertForQuestionAnswering, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/albert_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/albert_for_sequence_classification.py index c7aa03f15cc0eb..fa9e7b5168b709 100755 --- a/python/sparknlp/annotator/classifier_dl/albert_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/albert_for_sequence_classification.py @@ -178,7 +178,7 @@ def loadSavedModel(folder, spark_session): return AlbertForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="albert_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="albert_base_sequence_classifier_imdb", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -198,4 +198,4 @@ def pretrained(name="albert_base_sequence_classifier_imdb", lang="en", remote_lo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertForSequenceClassification, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(AlbertForSequenceClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/albert_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/albert_for_token_classification.py index 8c8e569c10ba15..814ec342a5589b 100755 --- a/python/sparknlp/annotator/classifier_dl/albert_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/albert_for_token_classification.py @@ -156,7 +156,7 @@ def loadSavedModel(folder, spark_session): return AlbertForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="albert_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="albert_base_token_classifier_conll03", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -176,4 +176,4 @@ def pretrained(name="albert_base_token_classifier_conll03", lang="en", remote_lo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertForTokenClassification, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(AlbertForTokenClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/albert_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/albert_for_zero_shot_classification.py index 9ffb4a1c5cc27c..4ea47dab83166e 100644 --- a/python/sparknlp/annotator/classifier_dl/albert_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/albert_for_zero_shot_classification.py @@ -188,7 +188,7 @@ def loadSavedModel(folder, spark_session): return AlbertForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="albert_zero_shot_classifier_onnx", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="albert_zero_shot_classifier_onnx", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -208,4 +208,4 @@ def pretrained(name="albert_zero_shot_classifier_onnx", lang="en", remote_loc=No The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertForZeroShotClassification, name, lang, remote_loc,engine ) \ No newline at end of file + return ResourceDownloader.downloadModel(AlbertForZeroShotClassification, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/bart_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/bart_for_zero_shot_classification.py index da45cb89289b00..8a05f20805b2d8 100644 --- a/python/sparknlp/annotator/classifier_dl/bart_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/bart_for_zero_shot_classification.py @@ -202,7 +202,7 @@ def loadSavedModel(folder, spark_session): return BartForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="bart_large_zero_shot_classifier_mnli", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="bart_large_zero_shot_classifier_mnli", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -222,4 +222,4 @@ def pretrained(name="bart_large_zero_shot_classifier_mnli", lang="en", remote_lo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BartForZeroShotClassification, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(BartForZeroShotClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/bert_for_multiple_choice.py b/python/sparknlp/annotator/classifier_dl/bert_for_multiple_choice.py index 2854d0b3f46c2c..045e8d64180b53 100644 --- a/python/sparknlp/annotator/classifier_dl/bert_for_multiple_choice.py +++ b/python/sparknlp/annotator/classifier_dl/bert_for_multiple_choice.py @@ -138,7 +138,7 @@ def loadSavedModel(folder, spark_session): return BertForMultipleChoice(java_model=jModel) @staticmethod - def pretrained(name="bert_base_uncased_multiple_choice", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="bert_base_uncased_multiple_choice", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -158,4 +158,4 @@ def pretrained(name="bert_base_uncased_multiple_choice", lang="en", remote_loc=N The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertForMultipleChoice, name, lang, remote_loc,engine ) \ No newline at end of file + return ResourceDownloader.downloadModel(BertForMultipleChoice, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/bert_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/bert_for_question_answering.py index fa1f07b5750b4d..27b61bdccdd953 100755 --- a/python/sparknlp/annotator/classifier_dl/bert_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/bert_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return BertForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="bert_base_cased_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="bert_base_cased_qa_squad2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="bert_base_cased_qa_squad2", lang="en", remote_loc=None,engi The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertForQuestionAnswering, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(BertForQuestionAnswering, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/bert_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/bert_for_sequence_classification.py index fae880b5bf5fa2..8ae3dbf185df6a 100755 --- a/python/sparknlp/annotator/classifier_dl/bert_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/bert_for_sequence_classification.py @@ -179,7 +179,7 @@ def loadSavedModel(folder, spark_session): return BertForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="bert_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="bert_base_sequence_classifier_imdb", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -199,4 +199,4 @@ def pretrained(name="bert_base_sequence_classifier_imdb", lang="en", remote_loc= The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertForSequenceClassification, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(BertForSequenceClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/bert_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/bert_for_token_classification.py index 5b9ff56b6fdfbe..bef1f945b7ba06 100755 --- a/python/sparknlp/annotator/classifier_dl/bert_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/bert_for_token_classification.py @@ -154,7 +154,7 @@ def loadSavedModel(folder, spark_session): return BertForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="bert_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="bert_base_token_classifier_conll03", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -174,4 +174,4 @@ def pretrained(name="bert_base_token_classifier_conll03", lang="en", remote_loc= The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertForTokenClassification, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(BertForTokenClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/bert_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/bert_for_zero_shot_classification.py index 900faea1b084cd..e2058c6c77f8c1 100755 --- a/python/sparknlp/annotator/classifier_dl/bert_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/bert_for_zero_shot_classification.py @@ -189,7 +189,7 @@ def loadSavedModel(folder, spark_session): return BertForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="bert_zero_shot_classifier_mnli", lang="xx", remote_loc=None,engine ="onnx"): + def pretrained(name="bert_zero_shot_classifier_mnli", lang="xx", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -209,4 +209,4 @@ def pretrained(name="bert_zero_shot_classifier_mnli", lang="xx", remote_loc=None The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertForZeroShotClassification, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(BertForZeroShotClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/camembert_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/camembert_for_question_answering.py index 938f4586899563..02828e034c5125 100755 --- a/python/sparknlp/annotator/classifier_dl/camembert_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/camembert_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return CamemBertForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="camembert_base_qa_fquad", lang="fr", remote_loc=None,engine ="onnx"): + def pretrained(name="camembert_base_qa_fquad", lang="fr", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="camembert_base_qa_fquad", lang="fr", remote_loc=None,engine The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CamemBertForQuestionAnswering, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(CamemBertForQuestionAnswering, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/camembert_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/camembert_for_sequence_classification.py index 28e1c36d50bdc8..305f9d9453baf0 100644 --- a/python/sparknlp/annotator/classifier_dl/camembert_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/camembert_for_sequence_classification.py @@ -182,7 +182,7 @@ def loadSavedModel(folder, spark_session): return CamemBertForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="camembert_base_sequence_classifier_allocine", lang="fr", remote_loc=None,engine ="onnx"): + def pretrained(name="camembert_base_sequence_classifier_allocine", lang="fr", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -202,4 +202,4 @@ def pretrained(name="camembert_base_sequence_classifier_allocine", lang="fr", re The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CamemBertForSequenceClassification, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(CamemBertForSequenceClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/camembert_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/camembert_for_token_classification.py index 94f466d26d7be5..19fd4780f8e583 100755 --- a/python/sparknlp/annotator/classifier_dl/camembert_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/camembert_for_token_classification.py @@ -150,7 +150,7 @@ def loadSavedModel(folder, spark_session): return CamemBertForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="camembert_base_token_classifier_wikiner", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="camembert_base_token_classifier_wikiner", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -170,4 +170,4 @@ def pretrained(name="camembert_base_token_classifier_wikiner", lang="en", remote The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CamemBertForTokenClassification, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(CamemBertForTokenClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/camembert_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/camembert_for_zero_shot_classification.py index 86564a358d9b4d..7b16c4475e5511 100644 --- a/python/sparknlp/annotator/classifier_dl/camembert_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/camembert_for_zero_shot_classification.py @@ -179,7 +179,7 @@ def loadSavedModel(folder, spark_session): return CamemBertForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="camembert_zero_shot_classifier_xnli_onnx", lang="fr", remote_loc=None,engine ="onnx"): + def pretrained(name="camembert_zero_shot_classifier_xnli_onnx", lang="fr", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -199,4 +199,4 @@ def pretrained(name="camembert_zero_shot_classifier_xnli_onnx", lang="fr", remot The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CamemBertForZeroShotClassification, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(CamemBertForZeroShotClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/classifier_dl.py b/python/sparknlp/annotator/classifier_dl/classifier_dl.py index fd3219c2437646..90b30db9543a8d 100755 --- a/python/sparknlp/annotator/classifier_dl/classifier_dl.py +++ b/python/sparknlp/annotator/classifier_dl/classifier_dl.py @@ -298,7 +298,7 @@ def setConfigProtoBytes(self, b): return self._set(configProtoBytes=b) @staticmethod - def pretrained(name="classifierdl_use_trec6", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="classifierdl_use_trec6", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -317,4 +317,4 @@ def pretrained(name="classifierdl_use_trec6", lang="en", remote_loc=None,engine The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ClassifierDLModel, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(ClassifierDLModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/deberta_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/deberta_for_question_answering.py index 7424a86c62ddd4..bc2c09c6a434a3 100755 --- a/python/sparknlp/annotator/classifier_dl/deberta_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/deberta_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return DeBertaForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="deberta_v3_xsmall_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="deberta_v3_xsmall_qa_squad2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="deberta_v3_xsmall_qa_squad2", lang="en", remote_loc=None,en The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DeBertaForQuestionAnswering, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(DeBertaForQuestionAnswering, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/deberta_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/deberta_for_sequence_classification.py index 268888c791f30c..c61c09d97975bb 100755 --- a/python/sparknlp/annotator/classifier_dl/deberta_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/deberta_for_sequence_classification.py @@ -175,7 +175,7 @@ def loadSavedModel(folder, spark_session): return DeBertaForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="deberta_v3_xsmall_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="deberta_v3_xsmall_sequence_classifier_imdb", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -195,4 +195,4 @@ def pretrained(name="deberta_v3_xsmall_sequence_classifier_imdb", lang="en", rem The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DeBertaForSequenceClassification, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(DeBertaForSequenceClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/deberta_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/deberta_for_token_classification.py index 60493c45f79dfe..436118521414fd 100755 --- a/python/sparknlp/annotator/classifier_dl/deberta_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/deberta_for_token_classification.py @@ -152,7 +152,7 @@ def loadSavedModel(folder, spark_session): return DeBertaForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="deberta_v3_xsmall_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="deberta_v3_xsmall_token_classifier_conll03", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -172,4 +172,4 @@ def pretrained(name="deberta_v3_xsmall_token_classifier_conll03", lang="en", rem The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DeBertaForTokenClassification, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(DeBertaForTokenClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/deberta_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/deberta_for_zero_shot_classification.py index c61d75684f2f57..6e8364cf8caaf4 100644 --- a/python/sparknlp/annotator/classifier_dl/deberta_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/deberta_for_zero_shot_classification.py @@ -172,7 +172,7 @@ def loadSavedModel(folder, spark_session): return DeBertaForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="deberta_base_zero_shot_classifier_mnli_anli_v3", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="deberta_base_zero_shot_classifier_mnli_anli_v3", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters ---------- @@ -190,4 +190,4 @@ def pretrained(name="deberta_base_zero_shot_classifier_mnli_anli_v3", lang="en", The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DeBertaForZeroShotClassification, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(DeBertaForZeroShotClassification, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/distil_bert_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/distil_bert_for_question_answering.py index fe8bdd5c49254c..15230c58b69791 100755 --- a/python/sparknlp/annotator/classifier_dl/distil_bert_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/distil_bert_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return DistilBertForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_cased_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="distilbert_base_cased_qa_squad2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="distilbert_base_cased_qa_squad2", lang="en", remote_loc=Non The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertForQuestionAnswering, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(DistilBertForQuestionAnswering, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/distil_bert_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/distil_bert_for_sequence_classification.py index 6aa1c861c27f0c..16a7222cb1b1a0 100755 --- a/python/sparknlp/annotator/classifier_dl/distil_bert_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/distil_bert_for_sequence_classification.py @@ -178,7 +178,7 @@ def loadSavedModel(folder, spark_session): return DistilBertForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="distilbert_base_sequence_classifier_imdb", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -198,4 +198,4 @@ def pretrained(name="distilbert_base_sequence_classifier_imdb", lang="en", remot The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertForSequenceClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(DistilBertForSequenceClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/distil_bert_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/distil_bert_for_token_classification.py index 53a3d39b127086..984f5dfd732179 100755 --- a/python/sparknlp/annotator/classifier_dl/distil_bert_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/distil_bert_for_token_classification.py @@ -152,7 +152,7 @@ def loadSavedModel(folder, spark_session): return DistilBertForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="distilbert_base_token_classifier_conll03", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -172,4 +172,4 @@ def pretrained(name="distilbert_base_token_classifier_conll03", lang="en", remot The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertForTokenClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(DistilBertForTokenClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/distil_bert_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/distil_bert_for_zero_shot_classification.py index c93adc650f02fe..803179b46f2327 100644 --- a/python/sparknlp/annotator/classifier_dl/distil_bert_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/distil_bert_for_zero_shot_classification.py @@ -188,7 +188,7 @@ def loadSavedModel(folder, spark_session): return DistilBertForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_zero_shot_classifier_uncased_mnli", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="distilbert_base_zero_shot_classifier_uncased_mnli", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -208,4 +208,4 @@ def pretrained(name="distilbert_base_zero_shot_classifier_uncased_mnli", lang="e The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertForZeroShotClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(DistilBertForZeroShotClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/distilbert_for_multiple_choice.py b/python/sparknlp/annotator/classifier_dl/distilbert_for_multiple_choice.py index d6bcf925f0cfe4..f76aa3859c307e 100644 --- a/python/sparknlp/annotator/classifier_dl/distilbert_for_multiple_choice.py +++ b/python/sparknlp/annotator/classifier_dl/distilbert_for_multiple_choice.py @@ -138,7 +138,7 @@ def loadSavedModel(folder, spark_session): return DistilBertForMultipleChoice(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_uncased_multiple_choice", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="distilbert_base_uncased_multiple_choice", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -158,4 +158,4 @@ def pretrained(name="distilbert_base_uncased_multiple_choice", lang="en", remote The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertForMultipleChoice, name, lang, remote_loc,engine ) \ No newline at end of file + return ResourceDownloader.downloadModel(DistilBertForMultipleChoice, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/longformer_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/longformer_for_question_answering.py index 5b9052ae445409..c6f96e43e0480d 100755 --- a/python/sparknlp/annotator/classifier_dl/longformer_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/longformer_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return LongformerForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="longformer_base_base_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="longformer_base_base_qa_squad2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="longformer_base_base_qa_squad2", lang="en", remote_loc=None The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LongformerForQuestionAnswering, name, lang, remote_loc,engine ) + return ResourceDownloader.downloadModel(LongformerForQuestionAnswering, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/longformer_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/longformer_for_sequence_classification.py index 24ee808c5fb057..869ea11540a3af 100755 --- a/python/sparknlp/annotator/classifier_dl/longformer_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/longformer_for_sequence_classification.py @@ -178,7 +178,7 @@ def loadSavedModel(folder, spark_session): return LongformerForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="longformer_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="longformer_base_sequence_classifier_imdb", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -198,4 +198,4 @@ def pretrained(name="longformer_base_sequence_classifier_imdb", lang="en", remot The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LongformerForSequenceClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(LongformerForSequenceClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/longformer_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/longformer_for_token_classification.py index 533d876a9fff60..b9d4a4b21178ce 100755 --- a/python/sparknlp/annotator/classifier_dl/longformer_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/longformer_for_token_classification.py @@ -153,7 +153,7 @@ def loadSavedModel(folder, spark_session): return LongformerForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="longformer_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="longformer_base_token_classifier_conll03", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -173,4 +173,4 @@ def pretrained(name="longformer_base_token_classifier_conll03", lang="en", remot The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LongformerForTokenClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(LongformerForTokenClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/mpnet_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/mpnet_for_question_answering.py index 840a95efad9cc6..1738ce0cfd7f8c 100755 --- a/python/sparknlp/annotator/classifier_dl/mpnet_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/mpnet_for_question_answering.py @@ -125,7 +125,7 @@ def loadSavedModel(folder, spark_session): return MPNetForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="mpnet_base_question_answering_squad2", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="mpnet_base_question_answering_squad2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -145,4 +145,4 @@ def pretrained(name="mpnet_base_question_answering_squad2", lang="en", remote_lo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MPNetForQuestionAnswering, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(MPNetForQuestionAnswering, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/mpnet_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/mpnet_for_sequence_classification.py index 8c53f8d0ee3074..0f943ab16364fb 100755 --- a/python/sparknlp/annotator/classifier_dl/mpnet_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/mpnet_for_sequence_classification.py @@ -165,7 +165,7 @@ def loadSavedModel(folder, spark_session): return MPNetForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="mpnet_sequence_classifier_ukr_message", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="mpnet_sequence_classifier_ukr_message", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -185,4 +185,4 @@ def pretrained(name="mpnet_sequence_classifier_ukr_message", lang="en", remote_l The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MPNetForSequenceClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(MPNetForSequenceClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/mpnet_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/mpnet_for_token_classification.py index 13a3d72055dafb..c3111e95dbf337 100644 --- a/python/sparknlp/annotator/classifier_dl/mpnet_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/mpnet_for_token_classification.py @@ -150,7 +150,7 @@ def loadSavedModel(folder, spark_session): return MPNetForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -170,4 +170,4 @@ def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None,en The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MPNetForTokenClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(MPNetForTokenClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/multi_classifier_dl.py b/python/sparknlp/annotator/classifier_dl/multi_classifier_dl.py index 3b0929a365aac4..51922a2afbd25f 100755 --- a/python/sparknlp/annotator/classifier_dl/multi_classifier_dl.py +++ b/python/sparknlp/annotator/classifier_dl/multi_classifier_dl.py @@ -372,7 +372,7 @@ def setConfigProtoBytes(self, b): return self._set(configProtoBytes=b) @staticmethod - def pretrained(name="multiclassifierdl_use_toxic", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="multiclassifierdl_use_toxic", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -392,4 +392,4 @@ def pretrained(name="multiclassifierdl_use_toxic", lang="en", remote_loc=None,en The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MultiClassifierDLModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(MultiClassifierDLModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/roberta_for_multiple_choice.py b/python/sparknlp/annotator/classifier_dl/roberta_for_multiple_choice.py index 74aa17c446a3b4..4e01000b83b7dd 100644 --- a/python/sparknlp/annotator/classifier_dl/roberta_for_multiple_choice.py +++ b/python/sparknlp/annotator/classifier_dl/roberta_for_multiple_choice.py @@ -138,7 +138,7 @@ def loadSavedModel(folder, spark_session): return RoBertaForMultipleChoice(java_model=jModel) @staticmethod - def pretrained(name="Roberta_base_uncased_multiple_choice", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="Roberta_base_uncased_multiple_choice", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -158,4 +158,4 @@ def pretrained(name="Roberta_base_uncased_multiple_choice", lang="en", remote_lo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaForMultipleChoice, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(RoBertaForMultipleChoice, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/classifier_dl/roberta_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/roberta_for_question_answering.py index 9fcaa3f0f32d59..27a6bfdb979ef4 100755 --- a/python/sparknlp/annotator/classifier_dl/roberta_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/roberta_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return RoBertaForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="roberta_base_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="roberta_base_qa_squad2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="roberta_base_qa_squad2", lang="en", remote_loc=None,engine The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaForQuestionAnswering, name, lang, remote_loc, engine) + return ResourceDownloader.downloadModel(RoBertaForQuestionAnswering, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/roberta_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/roberta_for_sequence_classification.py index 13cb43228b55f1..6b26347c4397ca 100755 --- a/python/sparknlp/annotator/classifier_dl/roberta_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/roberta_for_sequence_classification.py @@ -178,7 +178,7 @@ def loadSavedModel(folder, spark_session): return RoBertaForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="roberta_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="roberta_base_sequence_classifier_imdb", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -198,4 +198,4 @@ def pretrained(name="roberta_base_sequence_classifier_imdb", lang="en", remote_l The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaForSequenceClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(RoBertaForSequenceClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/roberta_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/roberta_for_token_classification.py index c8fdc4ac6a99af..6cc071a29092b0 100755 --- a/python/sparknlp/annotator/classifier_dl/roberta_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/roberta_for_token_classification.py @@ -166,7 +166,7 @@ def loadSavedModel(folder, spark_session): return RoBertaForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="roberta_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="roberta_base_token_classifier_conll03", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -186,4 +186,4 @@ def pretrained(name="roberta_base_token_classifier_conll03", lang="en", remote_l The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaForTokenClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(RoBertaForTokenClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/roberta_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/roberta_for_zero_shot_classification.py index bd26088ba1f92d..29097acd25c553 100644 --- a/python/sparknlp/annotator/classifier_dl/roberta_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/roberta_for_zero_shot_classification.py @@ -202,7 +202,7 @@ def loadSavedModel(folder, spark_session): return RoBertaForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="roberta_base_zero_shot_classifier_nli", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="roberta_base_zero_shot_classifier_nli", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -222,4 +222,4 @@ def pretrained(name="roberta_base_zero_shot_classifier_nli", lang="en", remote_l The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaForZeroShotClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(RoBertaForZeroShotClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/sentiment_dl.py b/python/sparknlp/annotator/classifier_dl/sentiment_dl.py index 6c2c533dd0c482..362df774e6d81a 100755 --- a/python/sparknlp/annotator/classifier_dl/sentiment_dl.py +++ b/python/sparknlp/annotator/classifier_dl/sentiment_dl.py @@ -356,7 +356,7 @@ def setThresholdLabel(self, p): return self._set(thresholdLabel=p) @staticmethod - def pretrained(name="sentimentdl_use_imdb", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="sentimentdl_use_imdb", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -375,4 +375,4 @@ def pretrained(name="sentimentdl_use_imdb", lang="en", remote_loc=None,engine =" The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SentimentDLModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(SentimentDLModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/tapas_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/tapas_for_question_answering.py index 12d84d2146d182..5911d94d2f92e2 100644 --- a/python/sparknlp/annotator/classifier_dl/tapas_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/tapas_for_question_answering.py @@ -147,7 +147,7 @@ def loadSavedModel(folder, spark_session): return TapasForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="table_qa_tapas_base_finetuned_wtq", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="table_qa_tapas_base_finetuned_wtq", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -167,4 +167,4 @@ def pretrained(name="table_qa_tapas_base_finetuned_wtq", lang="en", remote_loc=N The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(TapasForQuestionAnswering, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(TapasForQuestionAnswering, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_multiple_choice.py b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_multiple_choice.py index 88ba63d0f6ad14..8da691d35cc091 100644 --- a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_multiple_choice.py +++ b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_multiple_choice.py @@ -126,7 +126,7 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaForMultipleChoice(java_model=jModel) @staticmethod - def pretrained(name="xlm_roberta_base_mc", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="xlm_roberta_base_mc", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -146,4 +146,4 @@ def pretrained(name="xlm_roberta_base_mc", lang="en", remote_loc=None,engine ="o The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaForMultipleChoice, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(XlmRoBertaForMultipleChoice, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_question_answering.py b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_question_answering.py index 117977cf7fe58c..97a2a38c08df86 100755 --- a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_question_answering.py +++ b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_question_answering.py @@ -145,7 +145,7 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="xlm_roberta_base_qa_squad2", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="xlm_roberta_base_qa_squad2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -165,4 +165,4 @@ def pretrained(name="xlm_roberta_base_qa_squad2", lang="en", remote_loc=None,eng The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaForQuestionAnswering, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(XlmRoBertaForQuestionAnswering, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_sequence_classification.py index b0ce95801db775..5112fa01e367d0 100755 --- a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_sequence_classification.py @@ -178,7 +178,7 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="xlm_roberta_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="xlm_roberta_base_token_classifier_conll03", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -198,4 +198,4 @@ def pretrained(name="xlm_roberta_base_token_classifier_conll03", lang="en", remo The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaForSequenceClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(XlmRoBertaForSequenceClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_token_classification.py index 93c66592829367..50a3668a181cea 100755 --- a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_token_classification.py @@ -150,7 +150,7 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -170,4 +170,4 @@ def pretrained(name="mpnet_base_token_classifier", lang="en", remote_loc=None,en The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaForTokenClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(XlmRoBertaForTokenClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_zero_shot_classification.py b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_zero_shot_classification.py index 1cb1c10b713287..36670c155a2d14 100644 --- a/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/classifier_dl/xlm_roberta_for_zero_shot_classification.py @@ -202,7 +202,7 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="xlm_roberta_large_zero_shot_classifier_xnli_anli", lang="xx", remote_loc=None,engine ="onnx"): + def pretrained(name="xlm_roberta_large_zero_shot_classifier_xnli_anli", lang="xx", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -222,4 +222,4 @@ def pretrained(name="xlm_roberta_large_zero_shot_classifier_xnli_anli", lang="xx The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaForZeroShotClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(XlmRoBertaForZeroShotClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/xlnet_for_sequence_classification.py b/python/sparknlp/annotator/classifier_dl/xlnet_for_sequence_classification.py index 195f55a4af5b1e..8a6c6ce5111e16 100755 --- a/python/sparknlp/annotator/classifier_dl/xlnet_for_sequence_classification.py +++ b/python/sparknlp/annotator/classifier_dl/xlnet_for_sequence_classification.py @@ -178,7 +178,7 @@ def loadSavedModel(folder, spark_session): return XlnetForSequenceClassification(java_model=jModel) @staticmethod - def pretrained(name="xlnet_base_sequence_classifier_imdb", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="xlnet_base_sequence_classifier_imdb", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -198,4 +198,4 @@ def pretrained(name="xlnet_base_sequence_classifier_imdb", lang="en", remote_loc The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlnetForSequenceClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(XlnetForSequenceClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/classifier_dl/xlnet_for_token_classification.py b/python/sparknlp/annotator/classifier_dl/xlnet_for_token_classification.py index 01ab9a0807ea0d..e0d1730b557a5b 100755 --- a/python/sparknlp/annotator/classifier_dl/xlnet_for_token_classification.py +++ b/python/sparknlp/annotator/classifier_dl/xlnet_for_token_classification.py @@ -153,7 +153,7 @@ def loadSavedModel(folder, spark_session): return XlnetForTokenClassification(java_model=jModel) @staticmethod - def pretrained(name="xlnet_base_token_classifier_conll03", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="xlnet_base_token_classifier_conll03", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -173,4 +173,4 @@ def pretrained(name="xlnet_base_token_classifier_conll03", lang="en", remote_loc The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlnetForTokenClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(XlnetForTokenClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/coref/spanbert_coref.py b/python/sparknlp/annotator/coref/spanbert_coref.py index 07d9b25fcc617b..b0d9d4761e42e0 100644 --- a/python/sparknlp/annotator/coref/spanbert_coref.py +++ b/python/sparknlp/annotator/coref/spanbert_coref.py @@ -199,7 +199,7 @@ def loadSavedModel(folder, spark_session): return SpanBertCorefModel(java_model=jModel) @staticmethod - def pretrained(name="spanbert_base_coref", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="spanbert_base_coref", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -218,4 +218,4 @@ def pretrained(name="spanbert_base_coref", lang="en", remote_loc=None,engine="on The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SpanBertCorefModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(SpanBertCorefModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/cv/blip_for_question_answering.py b/python/sparknlp/annotator/cv/blip_for_question_answering.py index 98b54b9325fe5b..fe018c0e683bf2 100644 --- a/python/sparknlp/annotator/cv/blip_for_question_answering.py +++ b/python/sparknlp/annotator/cv/blip_for_question_answering.py @@ -149,7 +149,7 @@ def loadSavedModel(folder, spark_session): return BLIPForQuestionAnswering(java_model=jModel) @staticmethod - def pretrained(name="blip_vqa_base", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="blip_vqa_base", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -169,4 +169,4 @@ def pretrained(name="blip_vqa_base", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BLIPForQuestionAnswering, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(BLIPForQuestionAnswering, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/clip_for_zero_shot_classification.py b/python/sparknlp/annotator/cv/clip_for_zero_shot_classification.py index 836a078b79982e..775f61f7d4d9bf 100644 --- a/python/sparknlp/annotator/cv/clip_for_zero_shot_classification.py +++ b/python/sparknlp/annotator/cv/clip_for_zero_shot_classification.py @@ -170,7 +170,7 @@ def loadSavedModel(folder, spark_session): return CLIPForZeroShotClassification(java_model=jModel) @staticmethod - def pretrained(name="zero_shot_classifier_clip_vit_base_patch32", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="zero_shot_classifier_clip_vit_base_patch32", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -190,4 +190,4 @@ def pretrained(name="zero_shot_classifier_clip_vit_base_patch32", lang="en", rem The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CLIPForZeroShotClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(CLIPForZeroShotClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/cv/convnext_for_image_classification.py b/python/sparknlp/annotator/cv/convnext_for_image_classification.py index a2951a69447dad..d86b71a3e44899 100644 --- a/python/sparknlp/annotator/cv/convnext_for_image_classification.py +++ b/python/sparknlp/annotator/cv/convnext_for_image_classification.py @@ -245,7 +245,7 @@ def loadSavedModel(folder, spark_session): @staticmethod def pretrained(name="image_classifier_convnext_tiny_224_local", lang="en", - remote_loc=None,engine ="onnx"): + remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -266,4 +266,4 @@ def pretrained(name="image_classifier_convnext_tiny_224_local", lang="en", """ from sparknlp.pretrained import ResourceDownloader return ResourceDownloader.downloadModel(ConvNextForImageClassification, name, lang, - remote_loc,engine) + remote_loc) diff --git a/python/sparknlp/annotator/cv/florence2_transformer.py b/python/sparknlp/annotator/cv/florence2_transformer.py index f0dcc0d718dba7..4617faddd6164f 100644 --- a/python/sparknlp/annotator/cv/florence2_transformer.py +++ b/python/sparknlp/annotator/cv/florence2_transformer.py @@ -174,7 +174,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Florence2Transformer(java_model=jModel) @staticmethod - def pretrained(name="florence_2_base_ft_int4", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="florence_2_base_ft_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model.""" from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Florence2Transformer, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(Florence2Transformer, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/gemma3_for_multimodal.py b/python/sparknlp/annotator/cv/gemma3_for_multimodal.py index cbb3722503cde2..01045933e6928c 100644 --- a/python/sparknlp/annotator/cv/gemma3_for_multimodal.py +++ b/python/sparknlp/annotator/cv/gemma3_for_multimodal.py @@ -324,7 +324,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Gemma3ForMultiModal(java_model=jModel) @staticmethod - def pretrained(name="gemma_3_4b_it_int4", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="gemma_3_4b_it_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -343,4 +343,4 @@ def pretrained(name="gemma_3_4b_it_int4", lang="en", remote_loc=None,engine ="on The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Gemma3ForMultiModal, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(Gemma3ForMultiModal, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/internvl_for_multimodal.py b/python/sparknlp/annotator/cv/internvl_for_multimodal.py index 2dcaefe81dedf5..a39476f5558aa7 100644 --- a/python/sparknlp/annotator/cv/internvl_for_multimodal.py +++ b/python/sparknlp/annotator/cv/internvl_for_multimodal.py @@ -259,7 +259,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return InternVLForMultiModal(java_model=jModel) @staticmethod - def pretrained(name="internvl2_5_1b_int4", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="internvl2_5_1b_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters ---------- @@ -277,4 +277,4 @@ def pretrained(name="internvl2_5_1b_int4", lang="en", remote_loc=None,engine ="o The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(InternVLForMultiModal, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(InternVLForMultiModal, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/cv/janus_for_multimodal.py b/python/sparknlp/annotator/cv/janus_for_multimodal.py index cc998241c5c72c..0e70cfe0c3edc8 100644 --- a/python/sparknlp/annotator/cv/janus_for_multimodal.py +++ b/python/sparknlp/annotator/cv/janus_for_multimodal.py @@ -330,7 +330,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return JanusForMultiModal(java_model=jModel) @staticmethod - def pretrained(name="janus_1_3b_int4", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="janus_1_3b_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters ---------- @@ -348,4 +348,4 @@ def pretrained(name="janus_1_3b_int4", lang="en", remote_loc=None,engine ="onnx" The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(JanusForMultiModal, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(JanusForMultiModal, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/llava_for_multimodal.py b/python/sparknlp/annotator/cv/llava_for_multimodal.py index caa94539e93975..84050df19806c4 100644 --- a/python/sparknlp/annotator/cv/llava_for_multimodal.py +++ b/python/sparknlp/annotator/cv/llava_for_multimodal.py @@ -305,7 +305,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return LLAVAForMultiModal(java_model=jModel) @staticmethod - def pretrained(name="llava_1_5_7b_hf", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="llava_1_5_7b_hf", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -325,4 +325,4 @@ def pretrained(name="llava_1_5_7b_hf", lang="en", remote_loc=None,engine ="onnx" The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LLAVAForMultiModal, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(LLAVAForMultiModal, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/mllama_for_multimodal.py b/python/sparknlp/annotator/cv/mllama_for_multimodal.py index 619eb1fb0d534b..1a4939b739d957 100644 --- a/python/sparknlp/annotator/cv/mllama_for_multimodal.py +++ b/python/sparknlp/annotator/cv/mllama_for_multimodal.py @@ -317,7 +317,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return MLLamaForMultimodal(java_model=jModel) @staticmethod - def pretrained(name="llama_3_2_11b_vision_instruct_int4", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="llama_3_2_11b_vision_instruct_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -337,4 +337,4 @@ def pretrained(name="llama_3_2_11b_vision_instruct_int4", lang="en", remote_loc= The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MLLamaForMultimodal, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(MLLamaForMultimodal, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/paligemma_for_multimodal.py b/python/sparknlp/annotator/cv/paligemma_for_multimodal.py index 4dabf65cafe154..06537b6ce805dc 100644 --- a/python/sparknlp/annotator/cv/paligemma_for_multimodal.py +++ b/python/sparknlp/annotator/cv/paligemma_for_multimodal.py @@ -284,7 +284,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return PaliGemmaForMultiModal(java_model=jModel) @staticmethod - def pretrained(name="paligemma_3b_pt_224_int4", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="paligemma_3b_pt_224_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -304,5 +304,5 @@ def pretrained(name="paligemma_3b_pt_224_int4", lang="en", remote_loc=None,engin The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(PaliGemmaForMultiModal, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(PaliGemmaForMultiModal, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/cv/phi3_vision_for_multimodal.py b/python/sparknlp/annotator/cv/phi3_vision_for_multimodal.py index 6fc6a87cfb8b9b..55d3c8c7b1c7fd 100644 --- a/python/sparknlp/annotator/cv/phi3_vision_for_multimodal.py +++ b/python/sparknlp/annotator/cv/phi3_vision_for_multimodal.py @@ -305,7 +305,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Phi3Vision(java_model=jModel) @staticmethod - def pretrained(name="phi_3_vision_128k_instruct", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="phi_3_vision_128k_instruct", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -325,4 +325,4 @@ def pretrained(name="phi_3_vision_128k_instruct", lang="en", remote_loc=None,eng The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Phi3Vision, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(Phi3Vision, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/qwen2vl_transformer.py b/python/sparknlp/annotator/cv/qwen2vl_transformer.py index 8aab66e8cd0d30..51433aed7cc7e4 100644 --- a/python/sparknlp/annotator/cv/qwen2vl_transformer.py +++ b/python/sparknlp/annotator/cv/qwen2vl_transformer.py @@ -309,7 +309,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Qwen2VLTransformer(java_model=jModel) @staticmethod - def pretrained(name="qwen2_vl_2b_instruct_int4", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="qwen2_vl_2b_instruct_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -329,4 +329,4 @@ def pretrained(name="qwen2_vl_2b_instruct_int4", lang="en", remote_loc=None,engi The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Qwen2VLTransformer, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(Qwen2VLTransformer, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/smolvlm_transformer.py b/python/sparknlp/annotator/cv/smolvlm_transformer.py index 7f79d4aaa92b13..461dcc84dcb6b9 100644 --- a/python/sparknlp/annotator/cv/smolvlm_transformer.py +++ b/python/sparknlp/annotator/cv/smolvlm_transformer.py @@ -405,7 +405,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return SmolVLMTransformer(java_model=jModel) @staticmethod - def pretrained(name="smolvlm_instruct_int4", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="smolvlm_instruct_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters ---------- @@ -423,4 +423,4 @@ def pretrained(name="smolvlm_instruct_int4", lang="en", remote_loc=None,engine = The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SmolVLMTransformer, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(SmolVLMTransformer, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/cv/swin_for_image_classification.py b/python/sparknlp/annotator/cv/swin_for_image_classification.py index f0997eacc6f96d..958b8e4a2b8300 100644 --- a/python/sparknlp/annotator/cv/swin_for_image_classification.py +++ b/python/sparknlp/annotator/cv/swin_for_image_classification.py @@ -218,7 +218,7 @@ def loadSavedModel(folder, spark_session): @staticmethod def pretrained(name="image_classifier_swin_base_patch4_window7_224", lang="en", - remote_loc=None,engine ="onnx"): + remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -239,4 +239,4 @@ def pretrained(name="image_classifier_swin_base_patch4_window7_224", lang="en", """ from sparknlp.pretrained import ResourceDownloader return ResourceDownloader.downloadModel(SwinForImageClassification, name, lang, - remote_loc,engine) + remote_loc) diff --git a/python/sparknlp/annotator/cv/vision_encoder_decoder_for_image_captioning.py b/python/sparknlp/annotator/cv/vision_encoder_decoder_for_image_captioning.py index 4f44b950795a18..6fe6f81b5c9012 100644 --- a/python/sparknlp/annotator/cv/vision_encoder_decoder_for_image_captioning.py +++ b/python/sparknlp/annotator/cv/vision_encoder_decoder_for_image_captioning.py @@ -217,7 +217,7 @@ def loadSavedModel(folder, spark_session): return VisionEncoderDecoderForImageCaptioning(java_model=jModel) @staticmethod - def pretrained(name="image_captioning_vit_gpt2", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="image_captioning_vit_gpt2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -237,4 +237,4 @@ def pretrained(name="image_captioning_vit_gpt2", lang="en", remote_loc=None,engi The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(VisionEncoderDecoderForImageCaptioning, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(VisionEncoderDecoderForImageCaptioning, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/cv/vit_for_image_classification.py b/python/sparknlp/annotator/cv/vit_for_image_classification.py index 6d405d6e39ef9f..7ad99661e9b19b 100644 --- a/python/sparknlp/annotator/cv/vit_for_image_classification.py +++ b/python/sparknlp/annotator/cv/vit_for_image_classification.py @@ -194,7 +194,7 @@ def loadSavedModel(folder, spark_session): return ViTForImageClassification(java_model=jModel) @staticmethod - def pretrained(name="image_classifier_vit_base_patch16_224", lang="en", remote_loc=None,engine ="onnx"): + def pretrained(name="image_classifier_vit_base_patch16_224", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -214,4 +214,4 @@ def pretrained(name="image_classifier_vit_base_patch16_224", lang="en", remote_l The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ViTForImageClassification, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(ViTForImageClassification, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/dependency/dependency_parser.py b/python/sparknlp/annotator/dependency/dependency_parser.py index 9ddc39bab60ea4..6a0789ae5e416f 100755 --- a/python/sparknlp/annotator/dependency/dependency_parser.py +++ b/python/sparknlp/annotator/dependency/dependency_parser.py @@ -272,7 +272,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.parser.dep.Depende ) @staticmethod - def pretrained(name="dependency_conllu", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="dependency_conllu", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -291,4 +291,4 @@ def pretrained(name="dependency_conllu", lang="en", remote_loc=None,engine="onnx The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DependencyParserModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(DependencyParserModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/dependency/typed_dependency_parser.py b/python/sparknlp/annotator/dependency/typed_dependency_parser.py index 990edeb913bab9..8fb0476c29bd18 100755 --- a/python/sparknlp/annotator/dependency/typed_dependency_parser.py +++ b/python/sparknlp/annotator/dependency/typed_dependency_parser.py @@ -296,7 +296,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.parser.typdep.Type ) @staticmethod - def pretrained(name="dependency_typed_conllu", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="dependency_typed_conllu", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -315,4 +315,4 @@ def pretrained(name="dependency_typed_conllu", lang="en", remote_loc=None,engine The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(TypedDependencyParserModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(TypedDependencyParserModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/albert_embeddings.py b/python/sparknlp/annotator/embeddings/albert_embeddings.py index a87d488c627465..72eef1e82d87e4 100755 --- a/python/sparknlp/annotator/embeddings/albert_embeddings.py +++ b/python/sparknlp/annotator/embeddings/albert_embeddings.py @@ -208,7 +208,7 @@ def loadSavedModel(folder, spark_session): return AlbertEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="albert_base_uncased", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="albert_base_uncased", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -227,4 +227,4 @@ def pretrained(name="albert_base_uncased", lang="en", remote_loc=None,engine="on The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AlbertEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(AlbertEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/auto_gguf_embeddings.py b/python/sparknlp/annotator/embeddings/auto_gguf_embeddings.py index 8604900846107f..ef0d8aabd78601 100755 --- a/python/sparknlp/annotator/embeddings/auto_gguf_embeddings.py +++ b/python/sparknlp/annotator/embeddings/auto_gguf_embeddings.py @@ -509,7 +509,7 @@ def loadSavedModel(folder, spark_session): return AutoGGUFEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="Qwen3_Embedding_0.6B_Q8_0_gguf", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="Qwen3_Embedding_0.6B_Q8_0_gguf", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -530,7 +530,7 @@ def pretrained(name="Qwen3_Embedding_0.6B_Q8_0_gguf", lang="en", remote_loc=None from sparknlp.pretrained import ResourceDownloader return ResourceDownloader.downloadModel( - AutoGGUFEmbeddings, name, lang, remote_loc,engine + AutoGGUFEmbeddings, name, lang, remote_loc ) def close(self): diff --git a/python/sparknlp/annotator/embeddings/bert_embeddings.py b/python/sparknlp/annotator/embeddings/bert_embeddings.py index df5fde7824bb58..9fb7b40076ef28 100755 --- a/python/sparknlp/annotator/embeddings/bert_embeddings.py +++ b/python/sparknlp/annotator/embeddings/bert_embeddings.py @@ -186,7 +186,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return BertEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="small_bert_L2_768", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="small_bert_L2_768", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -205,4 +205,4 @@ def pretrained(name="small_bert_L2_768", lang="en", remote_loc=None,engine="onnx The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(BertEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/bert_sentence_embeddings.py b/python/sparknlp/annotator/embeddings/bert_sentence_embeddings.py index 05e84e6d27a142..c2545620bfb97f 100755 --- a/python/sparknlp/annotator/embeddings/bert_sentence_embeddings.py +++ b/python/sparknlp/annotator/embeddings/bert_sentence_embeddings.py @@ -202,7 +202,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return BertSentenceEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="sent_small_bert_L2_768", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="sent_small_bert_L2_768", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -221,4 +221,4 @@ def pretrained(name="sent_small_bert_L2_768", lang="en", remote_loc=None,engine= The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BertSentenceEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(BertSentenceEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/bge_embeddings.py b/python/sparknlp/annotator/embeddings/bge_embeddings.py index 9d5b2a5613ab02..555d3169a234ea 100644 --- a/python/sparknlp/annotator/embeddings/bge_embeddings.py +++ b/python/sparknlp/annotator/embeddings/bge_embeddings.py @@ -177,7 +177,7 @@ def loadSavedModel(folder, spark_session): return BGEEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="bge_small_en_v1.5", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="bge_small_en_v1.5", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -196,4 +196,4 @@ def pretrained(name="bge_small_en_v1.5", lang="en", remote_loc=None,engine="onnx The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BGEEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(BGEEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/camembert_embeddings.py b/python/sparknlp/annotator/embeddings/camembert_embeddings.py index b7112d4df7daaf..2461324b143b23 100755 --- a/python/sparknlp/annotator/embeddings/camembert_embeddings.py +++ b/python/sparknlp/annotator/embeddings/camembert_embeddings.py @@ -188,7 +188,7 @@ def loadSavedModel(folder, spark_session): return CamemBertEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="camembert_base", lang="fr", remote_loc=None,engine="onnx"): + def pretrained(name="camembert_base", lang="fr", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -207,4 +207,4 @@ def pretrained(name="camembert_base", lang="fr", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CamemBertEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(CamemBertEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/deberta_embeddings.py b/python/sparknlp/annotator/embeddings/deberta_embeddings.py index dd218d6cc7a668..24462a0022b724 100755 --- a/python/sparknlp/annotator/embeddings/deberta_embeddings.py +++ b/python/sparknlp/annotator/embeddings/deberta_embeddings.py @@ -186,7 +186,7 @@ def loadSavedModel(folder, spark_session): return DeBertaEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="deberta_v3_base", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="deberta_v3_base", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -205,4 +205,4 @@ def pretrained(name="deberta_v3_base", lang="en", remote_loc=None,engine="onnx") The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DeBertaEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(DeBertaEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/distil_bert_embeddings.py b/python/sparknlp/annotator/embeddings/distil_bert_embeddings.py index 31c6ef3e640177..6ae825aec01fde 100755 --- a/python/sparknlp/annotator/embeddings/distil_bert_embeddings.py +++ b/python/sparknlp/annotator/embeddings/distil_bert_embeddings.py @@ -199,7 +199,7 @@ def loadSavedModel(folder, spark_session): return DistilBertEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="distilbert_base_cased", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="distilbert_base_cased", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -218,4 +218,4 @@ def pretrained(name="distilbert_base_cased", lang="en", remote_loc=None,engine=" The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(DistilBertEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(DistilBertEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/doc2vec.py b/python/sparknlp/annotator/embeddings/doc2vec.py index 82aa3b927e504e..1bc6c7120b8e77 100755 --- a/python/sparknlp/annotator/embeddings/doc2vec.py +++ b/python/sparknlp/annotator/embeddings/doc2vec.py @@ -323,7 +323,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.embeddings.Doc2VecModel", jav ) @staticmethod - def pretrained(name="doc2vec_gigaword_300", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="doc2vec_gigaword_300", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -342,7 +342,7 @@ def pretrained(name="doc2vec_gigaword_300", lang="en", remote_loc=None,engine="o The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Doc2VecModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(Doc2VecModel, name, lang, remote_loc) def getVectors(self): """ diff --git a/python/sparknlp/annotator/embeddings/e5_embeddings.py b/python/sparknlp/annotator/embeddings/e5_embeddings.py index 57d55bef78fd2a..c6c9e7c0e20033 100644 --- a/python/sparknlp/annotator/embeddings/e5_embeddings.py +++ b/python/sparknlp/annotator/embeddings/e5_embeddings.py @@ -173,7 +173,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return E5Embeddings(java_model=jModel) @staticmethod - def pretrained(name="e5_small", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="e5_small", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -192,4 +192,4 @@ def pretrained(name="e5_small", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(E5Embeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(E5Embeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/e5v_embeddings.py b/python/sparknlp/annotator/embeddings/e5v_embeddings.py index a66be3ea414b79..e8ee518a40333e 100644 --- a/python/sparknlp/annotator/embeddings/e5v_embeddings.py +++ b/python/sparknlp/annotator/embeddings/e5v_embeddings.py @@ -117,7 +117,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return E5VEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="e5v_int4", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="e5v_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -135,4 +135,4 @@ def pretrained(name="e5v_int4", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(E5VEmbeddings, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(E5VEmbeddings, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/embeddings/elmo_embeddings.py b/python/sparknlp/annotator/embeddings/elmo_embeddings.py index f1a958abb314cb..46b9858e84f697 100755 --- a/python/sparknlp/annotator/embeddings/elmo_embeddings.py +++ b/python/sparknlp/annotator/embeddings/elmo_embeddings.py @@ -229,7 +229,7 @@ def loadSavedModel(folder, spark_session): return ElmoEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="elmo", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="elmo", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -248,4 +248,4 @@ def pretrained(name="elmo", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ElmoEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(ElmoEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/instructor_embeddings.py b/python/sparknlp/annotator/embeddings/instructor_embeddings.py index 9ddbc19b4ead7f..31ca3c7fd52723 100755 --- a/python/sparknlp/annotator/embeddings/instructor_embeddings.py +++ b/python/sparknlp/annotator/embeddings/instructor_embeddings.py @@ -182,7 +182,7 @@ def loadSavedModel(folder, spark_session): return InstructorEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="instructor_base", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="instructor_base", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -201,4 +201,4 @@ def pretrained(name="instructor_base", lang="en", remote_loc=None,engine="onnx") The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(InstructorEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(InstructorEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/longformer_embeddings.py b/python/sparknlp/annotator/embeddings/longformer_embeddings.py index 02c027345bd3bf..ee286a8b3ed0f9 100755 --- a/python/sparknlp/annotator/embeddings/longformer_embeddings.py +++ b/python/sparknlp/annotator/embeddings/longformer_embeddings.py @@ -189,7 +189,7 @@ def loadSavedModel(folder, spark_session): return LongformerEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="longformer_base_4096", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="longformer_base_4096", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -208,4 +208,4 @@ def pretrained(name="longformer_base_4096", lang="en", remote_loc=None,engine="o The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LongformerEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(LongformerEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/minilm_embeddings.py b/python/sparknlp/annotator/embeddings/minilm_embeddings.py index 870bf1cdd1a335..a569dfcef9e83c 100644 --- a/python/sparknlp/annotator/embeddings/minilm_embeddings.py +++ b/python/sparknlp/annotator/embeddings/minilm_embeddings.py @@ -167,7 +167,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return MiniLMEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="minilm_l6_v2", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="minilm_l6_v2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -186,4 +186,4 @@ def pretrained(name="minilm_l6_v2", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MiniLMEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(MiniLMEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/mpnet_embeddings.py b/python/sparknlp/annotator/embeddings/mpnet_embeddings.py index 9a3aa8a08ef198..0d72cf3c388319 100644 --- a/python/sparknlp/annotator/embeddings/mpnet_embeddings.py +++ b/python/sparknlp/annotator/embeddings/mpnet_embeddings.py @@ -170,7 +170,7 @@ def loadSavedModel(folder, spark_session): return MPNetEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="all_mpnet_base_v2", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="all_mpnet_base_v2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -189,4 +189,4 @@ def pretrained(name="all_mpnet_base_v2", lang="en", remote_loc=None,engine="onnx The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MPNetEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(MPNetEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/mxbai_embeddings.py b/python/sparknlp/annotator/embeddings/mxbai_embeddings.py index 7cc35e81c9a9c6..854e8cac4e7f39 100644 --- a/python/sparknlp/annotator/embeddings/mxbai_embeddings.py +++ b/python/sparknlp/annotator/embeddings/mxbai_embeddings.py @@ -162,7 +162,7 @@ def loadSavedModel(folder, spark_session): return MxbaiEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="mxbai_large_v1", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="mxbai_large_v1", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -181,4 +181,4 @@ def pretrained(name="mxbai_large_v1", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MxbaiEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(MxbaiEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/nomic_embeddings.py b/python/sparknlp/annotator/embeddings/nomic_embeddings.py index f3fd9be9acc87a..b80597cac937d1 100644 --- a/python/sparknlp/annotator/embeddings/nomic_embeddings.py +++ b/python/sparknlp/annotator/embeddings/nomic_embeddings.py @@ -159,7 +159,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return NomicEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="nomic_embed_v1", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="nomic_embed_v1", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -178,4 +178,4 @@ def pretrained(name="nomic_embed_v1", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(NomicEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(NomicEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/roberta_embeddings.py b/python/sparknlp/annotator/embeddings/roberta_embeddings.py index 865c448b471534..72d180ae18cce5 100755 --- a/python/sparknlp/annotator/embeddings/roberta_embeddings.py +++ b/python/sparknlp/annotator/embeddings/roberta_embeddings.py @@ -203,7 +203,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return RoBertaEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="roberta_base", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="roberta_base", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -222,4 +222,4 @@ def pretrained(name="roberta_base", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(RoBertaEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/roberta_sentence_embeddings.py b/python/sparknlp/annotator/embeddings/roberta_sentence_embeddings.py index 25926642cdce4f..8a6d02c245af40 100755 --- a/python/sparknlp/annotator/embeddings/roberta_sentence_embeddings.py +++ b/python/sparknlp/annotator/embeddings/roberta_sentence_embeddings.py @@ -169,7 +169,7 @@ def loadSavedModel(folder, spark_session): return RoBertaSentenceEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="sent_roberta_base", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="sent_roberta_base", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -188,4 +188,4 @@ def pretrained(name="sent_roberta_base", lang="en", remote_loc=None,engine="onnx The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(RoBertaSentenceEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(RoBertaSentenceEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/snowflake_embeddings.py b/python/sparknlp/annotator/embeddings/snowflake_embeddings.py index cdfaa3d66ad45a..2e13cd2596d6ab 100644 --- a/python/sparknlp/annotator/embeddings/snowflake_embeddings.py +++ b/python/sparknlp/annotator/embeddings/snowflake_embeddings.py @@ -180,7 +180,7 @@ def loadSavedModel(folder, spark_session): return SnowFlakeEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="snowflake_artic_m", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="snowflake_artic_m", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -199,4 +199,4 @@ def pretrained(name="snowflake_artic_m", lang="en", remote_loc=None,engine="onnx The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SnowFlakeEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(SnowFlakeEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/uae_embeddings.py b/python/sparknlp/annotator/embeddings/uae_embeddings.py index ea3e92ffe7267d..1a25d63ded20c8 100644 --- a/python/sparknlp/annotator/embeddings/uae_embeddings.py +++ b/python/sparknlp/annotator/embeddings/uae_embeddings.py @@ -189,7 +189,7 @@ def loadSavedModel(folder, spark_session): return UAEEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="uae_large_v1", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="uae_large_v1", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -208,4 +208,4 @@ def pretrained(name="uae_large_v1", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(UAEEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(UAEEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/universal_sentence_encoder.py b/python/sparknlp/annotator/embeddings/universal_sentence_encoder.py index 87956015d8ffc5..ae155bba436f8d 100755 --- a/python/sparknlp/annotator/embeddings/universal_sentence_encoder.py +++ b/python/sparknlp/annotator/embeddings/universal_sentence_encoder.py @@ -189,7 +189,7 @@ def loadSavedModel(folder, spark_session, loadsp=False): return UniversalSentenceEncoder(java_model=jModel) @staticmethod - def pretrained(name="tfhub_use", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="tfhub_use", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -208,4 +208,4 @@ def pretrained(name="tfhub_use", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(UniversalSentenceEncoder, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(UniversalSentenceEncoder, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/word2vec.py b/python/sparknlp/annotator/embeddings/word2vec.py index 5c93f5d08db46d..c9c9450f5ffb4e 100755 --- a/python/sparknlp/annotator/embeddings/word2vec.py +++ b/python/sparknlp/annotator/embeddings/word2vec.py @@ -324,7 +324,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.embeddings.Word2VecModel", ja ) @staticmethod - def pretrained(name="word2vec_gigaword_300", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="word2vec_gigaword_300", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -343,7 +343,7 @@ def pretrained(name="word2vec_gigaword_300", lang="en", remote_loc=None,engine=" The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Word2VecModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(Word2VecModel, name, lang, remote_loc) def getVectors(self): """ diff --git a/python/sparknlp/annotator/embeddings/word_embeddings.py b/python/sparknlp/annotator/embeddings/word_embeddings.py index 2c3f070ae84e08..986c2822c0f87a 100755 --- a/python/sparknlp/annotator/embeddings/word_embeddings.py +++ b/python/sparknlp/annotator/embeddings/word_embeddings.py @@ -348,7 +348,7 @@ def withCoverageColumn(dataset, embeddings_col, output_col='coverage'): return DataFrame(_EmbeddingsCoverageColumn(dataset, embeddings_col, output_col).apply(), dataset.sql_ctx) @staticmethod - def pretrained(name="glove_100d", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="glove_100d", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -367,7 +367,7 @@ def pretrained(name="glove_100d", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(WordEmbeddingsModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(WordEmbeddingsModel, name, lang, remote_loc) @staticmethod def loadStorage(path, spark, storage_ref): diff --git a/python/sparknlp/annotator/embeddings/xlm_roberta_embeddings.py b/python/sparknlp/annotator/embeddings/xlm_roberta_embeddings.py index 1c141e0ab935f2..76c33eb27f2eb0 100755 --- a/python/sparknlp/annotator/embeddings/xlm_roberta_embeddings.py +++ b/python/sparknlp/annotator/embeddings/xlm_roberta_embeddings.py @@ -203,7 +203,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return XlmRoBertaEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="xlm_roberta_base", lang="xx", remote_loc=None,engine="onnx"): + def pretrained(name="xlm_roberta_base", lang="xx", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -222,4 +222,4 @@ def pretrained(name="xlm_roberta_base", lang="xx", remote_loc=None,engine="onnx" The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(XlmRoBertaEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/xlm_roberta_sentence_embeddings.py b/python/sparknlp/annotator/embeddings/xlm_roberta_sentence_embeddings.py index 14a5ca4c5f44e1..89a5a7e86cc9b7 100755 --- a/python/sparknlp/annotator/embeddings/xlm_roberta_sentence_embeddings.py +++ b/python/sparknlp/annotator/embeddings/xlm_roberta_sentence_embeddings.py @@ -172,7 +172,7 @@ def loadSavedModel(folder, spark_session): return XlmRoBertaSentenceEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="sent_xlm_roberta_base", lang="xx", remote_loc=None,engine="onnx"): + def pretrained(name="sent_xlm_roberta_base", lang="xx", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -191,4 +191,4 @@ def pretrained(name="sent_xlm_roberta_base", lang="xx", remote_loc=None,engine=" The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlmRoBertaSentenceEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(XlmRoBertaSentenceEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/embeddings/xlnet_embeddings.py b/python/sparknlp/annotator/embeddings/xlnet_embeddings.py index 4dc1d942f379a3..44b406ceaa7b6b 100755 --- a/python/sparknlp/annotator/embeddings/xlnet_embeddings.py +++ b/python/sparknlp/annotator/embeddings/xlnet_embeddings.py @@ -205,7 +205,7 @@ def loadSavedModel(folder, spark_session): return XlnetEmbeddings(java_model=jModel) @staticmethod - def pretrained(name="xlnet_base_cased", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="xlnet_base_cased", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -224,4 +224,4 @@ def pretrained(name="xlnet_base_cased", lang="en", remote_loc=None,engine="onnx" The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(XlnetEmbeddings, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(XlnetEmbeddings, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/ld_dl/language_detector_dl.py b/python/sparknlp/annotator/ld_dl/language_detector_dl.py index fda4bf822fe837..462b321dd4c693 100755 --- a/python/sparknlp/annotator/ld_dl/language_detector_dl.py +++ b/python/sparknlp/annotator/ld_dl/language_detector_dl.py @@ -177,7 +177,7 @@ def setCoalesceSentences(self, value): return self._set(coalesceSentences=value) @staticmethod - def pretrained(name="ld_wiki_tatoeba_cnn_21", lang="xx", remote_loc=None,engine="onnx"): + def pretrained(name="ld_wiki_tatoeba_cnn_21", lang="xx", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -196,4 +196,4 @@ def pretrained(name="ld_wiki_tatoeba_cnn_21", lang="xx", remote_loc=None,engine= The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LanguageDetectorDL, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(LanguageDetectorDL, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/lemmatizer.py b/python/sparknlp/annotator/lemmatizer.py index 771c207ed44ae9..85d4dc9b3b7a97 100755 --- a/python/sparknlp/annotator/lemmatizer.py +++ b/python/sparknlp/annotator/lemmatizer.py @@ -228,7 +228,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.LemmatizerModel", ) @staticmethod - def pretrained(name="lemma_antbnc", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="lemma_antbnc", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -247,4 +247,4 @@ def pretrained(name="lemma_antbnc", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LemmatizerModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(LemmatizerModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/matcher/big_text_matcher.py b/python/sparknlp/annotator/matcher/big_text_matcher.py index 825b080d665b48..1718306eea8d17 100755 --- a/python/sparknlp/annotator/matcher/big_text_matcher.py +++ b/python/sparknlp/annotator/matcher/big_text_matcher.py @@ -234,7 +234,7 @@ def setCaseSensitive(self, v): return self._set(caseSensitive=v) @staticmethod - def pretrained(name, lang="en", remote_loc=None,engine="onnx"): + def pretrained(name, lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -253,7 +253,7 @@ def pretrained(name, lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(TextMatcherModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(TextMatcherModel, name, lang, remote_loc) @staticmethod def loadStorage(path, spark, storage_ref): diff --git a/python/sparknlp/annotator/matcher/text_matcher.py b/python/sparknlp/annotator/matcher/text_matcher.py index 28fcc3edc155a6..86a9dc26053e93 100755 --- a/python/sparknlp/annotator/matcher/text_matcher.py +++ b/python/sparknlp/annotator/matcher/text_matcher.py @@ -268,7 +268,7 @@ def setBuildFromTokens(self, b): return self._set(buildFromTokens=b) @staticmethod - def pretrained(name, lang="en", remote_loc=None,engine="onnx"): + def pretrained(name, lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -287,4 +287,4 @@ def pretrained(name, lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(TextMatcherModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(TextMatcherModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/ner/ner_crf.py b/python/sparknlp/annotator/ner/ner_crf.py index 8e2c58c257fc03..09b08222d7b8b1 100755 --- a/python/sparknlp/annotator/ner/ner_crf.py +++ b/python/sparknlp/annotator/ner/ner_crf.py @@ -375,7 +375,7 @@ def setIncludeConfidence(self, b): return self._set(includeConfidence=b) @staticmethod - def pretrained(name="ner_crf", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="ner_crf", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -394,4 +394,4 @@ def pretrained(name="ner_crf", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(NerCrfModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(NerCrfModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/ner/ner_dl.py b/python/sparknlp/annotator/ner/ner_dl.py index 028c4d72fbec95..194468b16496c1 100755 --- a/python/sparknlp/annotator/ner/ner_dl.py +++ b/python/sparknlp/annotator/ner/ner_dl.py @@ -569,7 +569,7 @@ def setIncludeAllConfidenceScores(self, value): return self._set(includeAllConfidenceScores=value) @staticmethod - def pretrained(name="ner_dl", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="ner_dl", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -588,4 +588,4 @@ def pretrained(name="ner_dl", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(NerDLModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(NerDLModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/ner/zero_shot_ner_model.py b/python/sparknlp/annotator/ner/zero_shot_ner_model.py index c1bf0294a17ead..91412258299775 100644 --- a/python/sparknlp/annotator/ner/zero_shot_ner_model.py +++ b/python/sparknlp/annotator/ner/zero_shot_ner_model.py @@ -161,9 +161,9 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.ner.dl.ZeroShotNer ) @staticmethod - def pretrained(name="zero_shot_ner_roberta", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="zero_shot_ner_roberta", lang="en", remote_loc=None): from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ZeroShotNerModel, name, lang, remote_loc,engine, + return ResourceDownloader.downloadModel(ZeroShotNerModel, name, lang, remote_loc, j_dwn='PythonResourceDownloader') @staticmethod diff --git a/python/sparknlp/annotator/pos/perceptron.py b/python/sparknlp/annotator/pos/perceptron.py index 5b8aa1d34cca8f..1c461df7f9bd94 100755 --- a/python/sparknlp/annotator/pos/perceptron.py +++ b/python/sparknlp/annotator/pos/perceptron.py @@ -241,7 +241,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.pos.perceptron.Per ) @staticmethod - def pretrained(name="pos_anc", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="pos_anc", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -260,4 +260,4 @@ def pretrained(name="pos_anc", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(PerceptronModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(PerceptronModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/sentence/sentence_detector_dl.py b/python/sparknlp/annotator/sentence/sentence_detector_dl.py index e6437bea34645b..6e7051e1a6dd59 100755 --- a/python/sparknlp/annotator/sentence/sentence_detector_dl.py +++ b/python/sparknlp/annotator/sentence/sentence_detector_dl.py @@ -445,7 +445,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.sentence_detector_ ) @staticmethod - def pretrained(name="sentence_detector_dl", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="sentence_detector_dl", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -464,4 +464,4 @@ def pretrained(name="sentence_detector_dl", lang="en", remote_loc=None,engine="o The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SentenceDetectorDLModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(SentenceDetectorDLModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/sentiment/vivekn_sentiment.py b/python/sparknlp/annotator/sentiment/vivekn_sentiment.py index 8e0452a3c7ee85..53211165910d35 100755 --- a/python/sparknlp/annotator/sentiment/vivekn_sentiment.py +++ b/python/sparknlp/annotator/sentiment/vivekn_sentiment.py @@ -220,7 +220,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.sda.vivekn.ViveknS ) @staticmethod - def pretrained(name="sentiment_vivekn", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="sentiment_vivekn", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -239,4 +239,4 @@ def pretrained(name="sentiment_vivekn", lang="en", remote_loc=None,engine="onnx" The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ViveknSentimentModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(ViveknSentimentModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/auto_gguf_model.py b/python/sparknlp/annotator/seq2seq/auto_gguf_model.py index 121db120a20cfe..2495afdced304a 100755 --- a/python/sparknlp/annotator/seq2seq/auto_gguf_model.py +++ b/python/sparknlp/annotator/seq2seq/auto_gguf_model.py @@ -277,7 +277,7 @@ def loadSavedModel(path, spark_session): return AutoGGUFModel(java_model=jModel) @staticmethod - def pretrained(name="Phi_4_mini_instruct_Q4_K_M_gguf", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="Phi_4_mini_instruct_Q4_K_M_gguf", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -296,7 +296,7 @@ def pretrained(name="Phi_4_mini_instruct_Q4_K_M_gguf", lang="en", remote_loc=Non The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AutoGGUFModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(AutoGGUFModel, name, lang, remote_loc) def close(self): """Closes the llama.cpp model backend freeing resources. The model is reloaded when used again. diff --git a/python/sparknlp/annotator/seq2seq/auto_gguf_reranker.py b/python/sparknlp/annotator/seq2seq/auto_gguf_reranker.py index 7fc1ebeaf62b97..075e45c67f5493 100755 --- a/python/sparknlp/annotator/seq2seq/auto_gguf_reranker.py +++ b/python/sparknlp/annotator/seq2seq/auto_gguf_reranker.py @@ -307,7 +307,7 @@ def loadSavedModel(folder, spark_session): return AutoGGUFReranker(java_model=jModel) @staticmethod - def pretrained(name="bge_reranker_v2_m3_Q4_K_M", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="bge_reranker_v2_m3_Q4_K_M", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -326,7 +326,7 @@ def pretrained(name="bge_reranker_v2_m3_Q4_K_M", lang="en", remote_loc=None,engi The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AutoGGUFReranker, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(AutoGGUFReranker, name, lang, remote_loc) def close(self): """Closes the llama.cpp model backend freeing resources. The model is reloaded when used again. diff --git a/python/sparknlp/annotator/seq2seq/auto_gguf_vision_model.py b/python/sparknlp/annotator/seq2seq/auto_gguf_vision_model.py index d3d9f3277a4050..a632567b8b9dd1 100755 --- a/python/sparknlp/annotator/seq2seq/auto_gguf_vision_model.py +++ b/python/sparknlp/annotator/seq2seq/auto_gguf_vision_model.py @@ -309,7 +309,7 @@ def loadSavedModel(modelPath, mmprojPath, spark_session): return AutoGGUFVisionModel(java_model=jModel) @staticmethod - def pretrained(name="Qwen2.5_VL_3B_Instruct_Q4_K_M_gguf", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="Qwen2.5_VL_3B_Instruct_Q4_K_M_gguf", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -328,7 +328,7 @@ def pretrained(name="Qwen2.5_VL_3B_Instruct_Q4_K_M_gguf", lang="en", remote_loc= The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(AutoGGUFVisionModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(AutoGGUFVisionModel, name, lang, remote_loc) def close(self): """Closes the llama.cpp model backend freeing resources. The model is reloaded when used again. diff --git a/python/sparknlp/annotator/seq2seq/bart_transformer.py b/python/sparknlp/annotator/seq2seq/bart_transformer.py index 0cdbc8aecd49ce..c9db8f039b43fe 100755 --- a/python/sparknlp/annotator/seq2seq/bart_transformer.py +++ b/python/sparknlp/annotator/seq2seq/bart_transformer.py @@ -398,7 +398,7 @@ def loadSavedModel(folder, spark_session, use_cache=False): return BartTransformer(java_model=jModel) @staticmethod - def pretrained(name="distilbart_xsum_12_6", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="distilbart_xsum_12_6", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -417,4 +417,4 @@ def pretrained(name="distilbart_xsum_12_6", lang="en", remote_loc=None,engine="o The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(BartTransformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(BartTransformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/cohere_transformer.py b/python/sparknlp/annotator/seq2seq/cohere_transformer.py index 4b29b24057d4dd..f72994860171a4 100644 --- a/python/sparknlp/annotator/seq2seq/cohere_transformer.py +++ b/python/sparknlp/annotator/seq2seq/cohere_transformer.py @@ -335,7 +335,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return CoHereTransformer(java_model=jModel) @staticmethod - def pretrained(name="c4ai_command_r_v01_int4", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="c4ai_command_r_v01_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -354,4 +354,4 @@ def pretrained(name="c4ai_command_r_v01_int4", lang="en", remote_loc=None,engine The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CoHereTransformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(CoHereTransformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/cpm_transformer.py b/python/sparknlp/annotator/seq2seq/cpm_transformer.py index c90114dc2d4b50..7da0e216686fd6 100644 --- a/python/sparknlp/annotator/seq2seq/cpm_transformer.py +++ b/python/sparknlp/annotator/seq2seq/cpm_transformer.py @@ -299,7 +299,7 @@ def loadSavedModel(folder, spark_session, use_openvino = False): return CPMTransformer(java_model=jModel) @staticmethod - def pretrained(name="mini_cpm_2b_8bit", lang="xx", remote_loc=None,engine="onnx"): + def pretrained(name="mini_cpm_2b_8bit", lang="xx", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -318,4 +318,4 @@ def pretrained(name="mini_cpm_2b_8bit", lang="xx", remote_loc=None,engine="onnx" The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(CPMTransformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(CPMTransformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/gpt2_transformer.py b/python/sparknlp/annotator/seq2seq/gpt2_transformer.py index 9adea769d91a98..346050c7ac0d4f 100755 --- a/python/sparknlp/annotator/seq2seq/gpt2_transformer.py +++ b/python/sparknlp/annotator/seq2seq/gpt2_transformer.py @@ -341,7 +341,7 @@ def loadSavedModel(folder, spark_session): return GPT2Transformer(java_model=jModel) @staticmethod - def pretrained(name="gpt2", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="gpt2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -360,4 +360,4 @@ def pretrained(name="gpt2", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(GPT2Transformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(GPT2Transformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/llama2_transformer.py b/python/sparknlp/annotator/seq2seq/llama2_transformer.py index dcaebd28658705..d2d973ee4d7ced 100644 --- a/python/sparknlp/annotator/seq2seq/llama2_transformer.py +++ b/python/sparknlp/annotator/seq2seq/llama2_transformer.py @@ -321,7 +321,7 @@ def loadSavedModel(folder, spark_session, use_openvino = False): return LLAMA2Transformer(java_model=jModel) @staticmethod - def pretrained(name="llama_2_7b_chat_hf_int4", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="llama_2_7b_chat_hf_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -340,4 +340,4 @@ def pretrained(name="llama_2_7b_chat_hf_int4", lang="en", remote_loc=None,engine The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LLAMA2Transformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(LLAMA2Transformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/llama3_transformer.py b/python/sparknlp/annotator/seq2seq/llama3_transformer.py index 9de976d5f45d1d..d4761bcadb56e0 100644 --- a/python/sparknlp/annotator/seq2seq/llama3_transformer.py +++ b/python/sparknlp/annotator/seq2seq/llama3_transformer.py @@ -359,7 +359,7 @@ def loadSavedModel(folder, spark_session, use_openvino = False): return LLAMA3Transformer(java_model=jModel) @staticmethod - def pretrained(name="llama_3_7b_instruct_hf_int4", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="llama_3_7b_instruct_hf_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -378,4 +378,4 @@ def pretrained(name="llama_3_7b_instruct_hf_int4", lang="en", remote_loc=None,en The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(LLAMA3Transformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(LLAMA3Transformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/m2m100_transformer.py b/python/sparknlp/annotator/seq2seq/m2m100_transformer.py index 3a3188474d095c..cadf438cbd372b 100644 --- a/python/sparknlp/annotator/seq2seq/m2m100_transformer.py +++ b/python/sparknlp/annotator/seq2seq/m2m100_transformer.py @@ -370,7 +370,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return M2M100Transformer(java_model=jModel) @staticmethod - def pretrained(name="m2m100_418M", lang="xx", remote_loc=None,engine="onnx"): + def pretrained(name="m2m100_418M", lang="xx", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -389,4 +389,4 @@ def pretrained(name="m2m100_418M", lang="xx", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(M2M100Transformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(M2M100Transformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/marian_transformer.py b/python/sparknlp/annotator/seq2seq/marian_transformer.py index b76425d5c14ed0..a8586af9a4fd72 100755 --- a/python/sparknlp/annotator/seq2seq/marian_transformer.py +++ b/python/sparknlp/annotator/seq2seq/marian_transformer.py @@ -352,7 +352,7 @@ def loadSavedModel(folder, spark_session): return MarianTransformer(java_model=jModel) @staticmethod - def pretrained(name="opus_mt_en_fr", lang="xx", remote_loc=None,engine="onnx"): + def pretrained(name="opus_mt_en_fr", lang="xx", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -371,4 +371,4 @@ def pretrained(name="opus_mt_en_fr", lang="xx", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MarianTransformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(MarianTransformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/mistral_transformer.py b/python/sparknlp/annotator/seq2seq/mistral_transformer.py index 709ba534db8650..1f810fe6305031 100644 --- a/python/sparknlp/annotator/seq2seq/mistral_transformer.py +++ b/python/sparknlp/annotator/seq2seq/mistral_transformer.py @@ -326,7 +326,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return MistralTransformer(java_model=jModel) @staticmethod - def pretrained(name="mistral_7b", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="mistral_7b", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -345,4 +345,4 @@ def pretrained(name="mistral_7b", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(MistralTransformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(MistralTransformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/nllb_transformer.py b/python/sparknlp/annotator/seq2seq/nllb_transformer.py index e4da9f1327028f..e22efe1e3123ef 100644 --- a/python/sparknlp/annotator/seq2seq/nllb_transformer.py +++ b/python/sparknlp/annotator/seq2seq/nllb_transformer.py @@ -398,7 +398,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return NLLBTransformer(java_model=jModel) @staticmethod - def pretrained(name="nllb_distilled_600M_8int", lang="xx", remote_loc=None,engine="onnx"): + def pretrained(name="nllb_distilled_600M_8int", lang="xx", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -417,4 +417,4 @@ def pretrained(name="nllb_distilled_600M_8int", lang="xx", remote_loc=None,engin The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(NLLBTransformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(NLLBTransformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/olmo_transformer.py b/python/sparknlp/annotator/seq2seq/olmo_transformer.py index b8e2c54dd81c35..eb1b63d71cdcf1 100644 --- a/python/sparknlp/annotator/seq2seq/olmo_transformer.py +++ b/python/sparknlp/annotator/seq2seq/olmo_transformer.py @@ -304,7 +304,7 @@ def loadSavedModel(folder, spark_session): return OLMoTransformer(java_model=jModel) @staticmethod - def pretrained(name="olmo_1b_int4", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="olmo_1b_int4", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -323,4 +323,4 @@ def pretrained(name="olmo_1b_int4", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(OLMoTransformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(OLMoTransformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/phi2_transformer.py b/python/sparknlp/annotator/seq2seq/phi2_transformer.py index 7e1a032ce0dabe..d2eaaad2b960e7 100644 --- a/python/sparknlp/annotator/seq2seq/phi2_transformer.py +++ b/python/sparknlp/annotator/seq2seq/phi2_transformer.py @@ -304,7 +304,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Phi2Transformer(java_model=jModel) @staticmethod - def pretrained(name="phi2", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="phi2", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -323,4 +323,4 @@ def pretrained(name="phi2", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Phi2Transformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(Phi2Transformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/phi3_transformer.py b/python/sparknlp/annotator/seq2seq/phi3_transformer.py index 530a9807fd15a8..98a28eeac47b96 100644 --- a/python/sparknlp/annotator/seq2seq/phi3_transformer.py +++ b/python/sparknlp/annotator/seq2seq/phi3_transformer.py @@ -308,7 +308,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return Phi3Transformer(java_model=jModel) @staticmethod - def pretrained(name="phi_3_mini_128k_instruct", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="phi_3_mini_128k_instruct", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -327,4 +327,4 @@ def pretrained(name="phi_3_mini_128k_instruct", lang="en", remote_loc=None,engin The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Phi3Transformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(Phi3Transformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/phi4_transformer.py b/python/sparknlp/annotator/seq2seq/phi4_transformer.py index f863b76bc51579..2db4a4712cefb7 100644 --- a/python/sparknlp/annotator/seq2seq/phi4_transformer.py +++ b/python/sparknlp/annotator/seq2seq/phi4_transformer.py @@ -365,7 +365,7 @@ def loadSavedModel(folder, spark_session, use_openvino = False): return Phi4Transformer(java_model=jModel) @staticmethod - def pretrained(name="phi_4_mini_instruct_int8_openvino", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="phi_4_mini_instruct_int8_openvino", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -384,4 +384,4 @@ def pretrained(name="phi_4_mini_instruct_int8_openvino", lang="en", remote_loc=N The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(Phi4Transformer, name, lang, remote_loc,engine) \ No newline at end of file + return ResourceDownloader.downloadModel(Phi4Transformer, name, lang, remote_loc) \ No newline at end of file diff --git a/python/sparknlp/annotator/seq2seq/qwen_transformer.py b/python/sparknlp/annotator/seq2seq/qwen_transformer.py index ac73488661ef7c..9f2257ad1edd98 100644 --- a/python/sparknlp/annotator/seq2seq/qwen_transformer.py +++ b/python/sparknlp/annotator/seq2seq/qwen_transformer.py @@ -318,7 +318,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return QwenTransformer(java_model=jModel) @staticmethod - def pretrained(name="qwen_7.5b_chat", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="qwen_7.5b_chat", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -337,4 +337,4 @@ def pretrained(name="qwen_7.5b_chat", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(QwenTransformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(QwenTransformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/starcoder_transformer.py b/python/sparknlp/annotator/seq2seq/starcoder_transformer.py index b1850c28f2ca5b..3c87cb653682cc 100644 --- a/python/sparknlp/annotator/seq2seq/starcoder_transformer.py +++ b/python/sparknlp/annotator/seq2seq/starcoder_transformer.py @@ -313,7 +313,7 @@ def loadSavedModel(folder, spark_session, use_openvino=False): return StarCoderTransformer(java_model=jModel) @staticmethod - def pretrained(name="starcoder", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="starcoder", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -332,4 +332,4 @@ def pretrained(name="starcoder", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(StarCoderTransformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(StarCoderTransformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/seq2seq/t5_transformer.py b/python/sparknlp/annotator/seq2seq/t5_transformer.py index c3ad0f8b40bfe1..8191ba57fe5a62 100755 --- a/python/sparknlp/annotator/seq2seq/t5_transformer.py +++ b/python/sparknlp/annotator/seq2seq/t5_transformer.py @@ -403,7 +403,7 @@ def loadSavedModel(folder, spark_session): return T5Transformer(java_model=jModel) @staticmethod - def pretrained(name="t5_small", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="t5_small", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -422,4 +422,4 @@ def pretrained(name="t5_small", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(T5Transformer, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(T5Transformer, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/spell_check/context_spell_checker.py b/python/sparknlp/annotator/spell_check/context_spell_checker.py index 039d0da86d6afd..a7ff483cab6145 100755 --- a/python/sparknlp/annotator/spell_check/context_spell_checker.py +++ b/python/sparknlp/annotator/spell_check/context_spell_checker.py @@ -889,7 +889,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.spell.context.Cont ) @staticmethod - def pretrained(name="spellcheck_dl", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="spellcheck_dl", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -908,4 +908,4 @@ def pretrained(name="spellcheck_dl", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(ContextSpellCheckerModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(ContextSpellCheckerModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/spell_check/norvig_sweeting.py b/python/sparknlp/annotator/spell_check/norvig_sweeting.py index 1918fee599e7d0..74135d25fdfdeb 100755 --- a/python/sparknlp/annotator/spell_check/norvig_sweeting.py +++ b/python/sparknlp/annotator/spell_check/norvig_sweeting.py @@ -335,7 +335,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.spell.norvig.Norvi ) @staticmethod - def pretrained(name="spellcheck_norvig", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="spellcheck_norvig", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -354,5 +354,5 @@ def pretrained(name="spellcheck_norvig", lang="en", remote_loc=None,engine="onnx The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(NorvigSweetingModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(NorvigSweetingModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/spell_check/symmetric_delete.py b/python/sparknlp/annotator/spell_check/symmetric_delete.py index ce216d0ff92883..34862efb9b35a3 100755 --- a/python/sparknlp/annotator/spell_check/symmetric_delete.py +++ b/python/sparknlp/annotator/spell_check/symmetric_delete.py @@ -276,7 +276,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.spell.symmetric.Sy ) @staticmethod - def pretrained(name="spellcheck_sd", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="spellcheck_sd", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -295,5 +295,5 @@ def pretrained(name="spellcheck_sd", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(SymmetricDeleteModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(SymmetricDeleteModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/stop_words_cleaner.py b/python/sparknlp/annotator/stop_words_cleaner.py index 6caf125de04bdc..63536d394d4b18 100755 --- a/python/sparknlp/annotator/stop_words_cleaner.py +++ b/python/sparknlp/annotator/stop_words_cleaner.py @@ -168,7 +168,7 @@ def loadDefaultStopWords(language="english"): return list(stopWordsObj.loadDefaultStopWords(language)) @staticmethod - def pretrained(name="stopwords_en", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="stopwords_en", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -187,4 +187,4 @@ def pretrained(name="stopwords_en", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(StopWordsCleaner, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(StopWordsCleaner, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/token/tokenizer.py b/python/sparknlp/annotator/token/tokenizer.py index abfd76e482f806..dd4113815d5a91 100755 --- a/python/sparknlp/annotator/token/tokenizer.py +++ b/python/sparknlp/annotator/token/tokenizer.py @@ -539,7 +539,7 @@ def addSplitChars(self, value): return self._set(splitChars=split_chars) @staticmethod - def pretrained(name="token_rules", lang="en", remote_loc=None,engine="onnx"): + def pretrained(name="token_rules", lang="en", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -558,4 +558,4 @@ def pretrained(name="token_rules", lang="en", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(TokenizerModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(TokenizerModel, name, lang, remote_loc) diff --git a/python/sparknlp/annotator/ws/word_segmenter.py b/python/sparknlp/annotator/ws/word_segmenter.py index 2a6b8eb016bbe1..188de3bb21b388 100755 --- a/python/sparknlp/annotator/ws/word_segmenter.py +++ b/python/sparknlp/annotator/ws/word_segmenter.py @@ -407,7 +407,7 @@ def __init__(self, classname="com.johnsnowlabs.nlp.annotators.ws.WordSegmenterMo ) @staticmethod - def pretrained(name="wordseg_pku", lang="zh", remote_loc=None,engine="onnx"): + def pretrained(name="wordseg_pku", lang="zh", remote_loc=None): """Downloads and loads a pretrained model. Parameters @@ -426,4 +426,4 @@ def pretrained(name="wordseg_pku", lang="zh", remote_loc=None,engine="onnx"): The restored model """ from sparknlp.pretrained import ResourceDownloader - return ResourceDownloader.downloadModel(WordSegmenterModel, name, lang, remote_loc,engine) + return ResourceDownloader.downloadModel(WordSegmenterModel, name, lang, remote_loc) diff --git a/python/sparknlp/common/properties.py b/python/sparknlp/common/properties.py index 7ed53ebe9a8e09..30277c4bf40b90 100644 --- a/python/sparknlp/common/properties.py +++ b/python/sparknlp/common/properties.py @@ -502,6 +502,29 @@ def getEngine(self): """ return self.getOrDefault(self.engine) + @classmethod + def pretrainedEngine(cls, name: str = "default", lang: str = "en", remote_loc: str = None, engine="onnx"): + """Downloads and loads a pretrained model. + + Parameters + ---------- + name : str, optional + The name of the pretrained model, by default "default" + lang : str, optional + The language of the pretrained model, by default "en" + remote_loc : str, optional + Remote location of the model, by default None + engine : str, optional + The Deep Learning engine used for this model, by default "onnx" + + Returns + ------- + AnnotatorModel + Pretrained model + """ + from sparknlp.pretrained import ResourceDownloader + return ResourceDownloader.downloadModel(cls, name, lang, remote_loc, engine) + class HasCandidateLabelsProperties: candidateLabels = Param(Params._dummy(), "candidateLabels", diff --git a/python/test/annotator/classifier_dl/distilbert_for_token_classification_test.py b/python/test/annotator/classifier_dl/distilbert_for_token_classification_test.py index b196fc68e5e433..10a42ed64f8c26 100644 --- a/python/test/annotator/classifier_dl/distilbert_for_token_classification_test.py +++ b/python/test/annotator/classifier_dl/distilbert_for_token_classification_test.py @@ -28,8 +28,8 @@ def setUp(self): self.data = SparkContextForTest.spark.read.option("header", "true") \ .csv(path="file:///" + os.getcwd() + "/../src/test/resources/embeddings/sentence_embeddings.csv") - self.tested_annotator = DistilBertForTokenClassification.pretrained("distilbert_token_classifier_keyphrase_extraction_inspec",engine="openvino") \ - .setInputCols(["document", "token"]) \ + self.tested_annotator = DistilBertForTokenClassification.pretrained() \ + .setInputCols(["document", "token"]) \ .setOutputCol("ner") def test_run(self): @@ -50,5 +50,5 @@ def test_run(self): model = pipeline.fit(self.data) model.transform(self.data).show() - #print(self.classifier.getClasses()) - #print(self.classifier.getBatchSize()) + print(self.classifier.getClasses()) + print(self.classifier.getBatchSize()) diff --git a/python/test/annotator/embeddings/camembert_embeddings_test.py b/python/test/annotator/embeddings/camembert_embeddings_test.py index 94576cdd546e9a..a6d72a797eb4a8 100644 --- a/python/test/annotator/embeddings/camembert_embeddings_test.py +++ b/python/test/annotator/embeddings/camembert_embeddings_test.py @@ -50,7 +50,7 @@ def test_run(self): model.transform(self.data).show() def test_perferred_engine(self): - model_onnx = CamemBertEmbeddings.pretrained("camembert_base","fr",engine= "onnx") - model_tensorflow = CamemBertEmbeddings.pretrained("camembert_base","fr",engine= "tensorflow") + model_onnx = CamemBertEmbeddings.pretrainedEngine("camembert_base","fr",engine= "onnx") + model_tensorflow = CamemBertEmbeddings.pretrainedEngine("camembert_base","fr",engine= "tensorflow") self.assertEqual(model_onnx.getEngine(), "onnx") self.assertEqual(model_tensorflow.getEngine(), "tensorflow") From e34057c2acbfe596e8ced868b67d5238f4e76fa6 Mon Sep 17 00:00:00 2001 From: ahmedlone127 Date: Sat, 29 Nov 2025 04:38:59 +0500 Subject: [PATCH 8/8] SPARKNLP-1303 adding scala test cases to test new preferred engine behaviour --- .../nlp/pretrained/ResourceMetadata.scala | 24 ++++---- .../test_engine_metadata.json | 6 ++ .../nlp/pretrained/ResourceMedataTest.scala | 59 +++++++++++++++++++ 3 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 src/test/resources/resource-downloader/test_engine_metadata.json diff --git a/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceMetadata.scala b/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceMetadata.scala index 50dd946fe0fd84..a90463bedfdd6f 100644 --- a/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceMetadata.scala +++ b/src/main/scala/com/johnsnowlabs/nlp/pretrained/ResourceMetadata.scala @@ -109,17 +109,19 @@ object ResourceMetadata { candidates: List[ResourceMetadata], request: ResourceRequest): Option[ResourceMetadata] = { - val compatibleCandidates = candidates.filter(item => - item.readyToUse && - item.libVersion.isDefined && - item.sparkVersion.isDefined && - item.name == request.name && - (request.annotator.isEmpty || item.annotator.isEmpty || - request.annotator.get.equalsIgnoreCase(item.annotator.get)) && - (request.language.isEmpty || item.language.isEmpty || - request.language.get == item.language.get) && - Version.isCompatible(request.libVersion, item.libVersion) && - Version.isCompatible(request.sparkVersion, item.sparkVersion)) + val excludedAnnotators = Array("AutoGGUFModel","AutoGGUFVisionModel") + + val compatibleCandidates = candidates.filter(item => + item.readyToUse && + item.libVersion.isDefined && + item.sparkVersion.isDefined && + item.name == request.name & + (request.annotator.isEmpty || item.annotator.isEmpty || excludedAnnotators.contains(item.annotator.get) || + request.annotator.get.equalsIgnoreCase(item.annotator.get)) && + (request.language.isEmpty || item.language.isEmpty || + request.language.get == item.language.get) && + Version.isCompatible(request.libVersion, item.libVersion) && + Version.isCompatible(request.sparkVersion, item.sparkVersion)) val defaultPriority = Seq("onnx", "tensorflow", "openvino", "unk") diff --git a/src/test/resources/resource-downloader/test_engine_metadata.json b/src/test/resources/resource-downloader/test_engine_metadata.json new file mode 100644 index 00000000000000..6fff859130e734 --- /dev/null +++ b/src/test/resources/resource-downloader/test_engine_metadata.json @@ -0,0 +1,6 @@ +{"name": "bert_base_uncased", "language": "en", "libVersion": {"parts": [1, 5]}, "sparkVersion": {"parts": [2]}, "readyToUse": true, "time": "2018-03-27T19:57:33.497Z", "isZipped": true, "checksum": "", "annotator": "BertEmbeddings","engine":"unk"} +{"name": "bert_base_uncased", "language": "en", "libVersion": {"parts": [6, 1, 5]}, "sparkVersion": {"parts": [3]}, "readyToUse": true, "time": "2025-03-27T20:09:07.222Z", "isZipped": true, "checksum": "", "annotator": "BertEmbeddings","engine":"onnx"} +{"name": "bert_base_uncased", "language": "en", "libVersion": {"parts": [6, 1, 5]}, "sparkVersion": {"parts": [3]}, "readyToUse": true, "time": "2025-03-27T20:09:07.222Z", "isZipped": true, "checksum": "", "annotator": "BertEmbeddings","engine":"openvino"} +{"name": "bert_base_uncased", "language": "en", "libVersion": {"parts": [3, 3, 4]}, "sparkVersion": {"parts": [3]}, "readyToUse": true, "time": "2025-03-27T20:09:07.222Z", "isZipped": true, "checksum": "", "annotator": "BertEmbeddings","engine":"tensorflow"} +{"name": "bert_base_uncased", "language": "en", "libVersion": {"parts": [6, 2, 0]}, "sparkVersion": {"parts": [3]}, "readyToUse": true, "time": "2025-10-27T20:09:07.222Z", "isZipped": true, "checksum": "", "annotator": "BertSentenceEmbeddings","engine":"tensorflow"} +{"name": "testannotator", "language": "en", "libVersion": {"parts": [6, 0, 0]}, "sparkVersion": {"parts": [3]}, "readyToUse": true, "time": "2025-10-27T20:09:07.222Z", "isZipped": true, "checksum": "", "annotator": "BertSentenceEmbeddings","engine":"unk"} diff --git a/src/test/scala/com/johnsnowlabs/nlp/pretrained/ResourceMedataTest.scala b/src/test/scala/com/johnsnowlabs/nlp/pretrained/ResourceMedataTest.scala index 5035fed077bf1c..d821775e0f2e38 100644 --- a/src/test/scala/com/johnsnowlabs/nlp/pretrained/ResourceMedataTest.scala +++ b/src/test/scala/com/johnsnowlabs/nlp/pretrained/ResourceMedataTest.scala @@ -198,6 +198,65 @@ class ResourceMedataTest extends AnyFlatSpec { assert(versions.get.time == expectedTimestamp) } + it should "get correct model if two models have same name but belong to different annotators" in { + val resourcePath = "src/test/resources/resource-downloader/test_engine_metadata.json" + val mockResourceDownloader: MockResourceDownloader = new MockResourceDownloader(resourcePath) + val resourceMetadata = mockResourceDownloader.resources + val resourceRequest = ResourceRequest( + "bert_base_uncased", + libVersion = Version(List(6, 0, 0)), + sparkVersion = Version(List(3, 0)), + annotator = Some("BertEmbeddings")) + + val expectedAnnotator = "BertEmbeddings" + + val versions = ResourceMetadata.resolveResource(resourceMetadata, resourceRequest) + + assert(versions.get.annotator.get == expectedAnnotator) + } + + it should "get correct preferred engine in case model has preferred engine available" in { + val resourcePath = "src/test/resources/resource-downloader/test_engine_metadata.json" + val mockResourceDownloader: MockResourceDownloader = new MockResourceDownloader(resourcePath) + val resourceMetadata = mockResourceDownloader.resources + val resourceRequest = ResourceRequest( + "bert_base_uncased", + libVersion = Version(List(6, 0, 0)), + sparkVersion = Version(List(3, 0)), + annotator = Some("BertEmbeddings"), + engine = Some("tensorflow")) + + val expectedAnnotator = "BertEmbeddings" + val expectedEngine = "tensorflow" + + val versions = ResourceMetadata.resolveResource(resourceMetadata, resourceRequest) + + assert(versions.get.annotator.get == expectedAnnotator) + assert(versions.get.engine.get == expectedEngine) + } + + + it should "fall back to other model variant if preferred engine does not exist" in { + val resourcePath = "src/test/resources/resource-downloader/test_engine_metadata.json" + val mockResourceDownloader: MockResourceDownloader = new MockResourceDownloader(resourcePath) + val resourceMetadata = mockResourceDownloader.resources + val resourceRequest = ResourceRequest( + "testannotator", + libVersion = Version(List(6, 2, 0)), + sparkVersion = Version(List(3, 0)), + annotator = Some("BertSentenceEmbeddings"), + engine = Some("tensorflow")) + + val expectedAnnotator = "BertSentenceEmbeddings" + val expectedEngine = "unk" + + val versions = ResourceMetadata.resolveResource(resourceMetadata, resourceRequest) + + assert(versions.get.annotator.get == expectedAnnotator) + assert(versions.get.engine.get == expectedEngine) + } + + private def getTimestamp(date: String): Timestamp = { val UTC = TimeZone.getTimeZone("UTC") val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")