From 692a4b06d312bd6c7f0461d23f32051267094bcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 7 Jun 2023 19:54:21 +0200 Subject: [PATCH 01/19] Create master.yml --- .github/workflows/master.yml | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/master.yml diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml new file mode 100644 index 0000000..6e87551 --- /dev/null +++ b/.github/workflows/master.yml @@ -0,0 +1,59 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install . + pip install -r requirements-test.txt + pip install codecov + python -m spylon_kernel install --user + + - name: Test with pytest + run: | + python run_tests.py -vxrs --capture=sys --color=yes + python setup.py sdist + pip install --no-binary :all: dist/*.tar.gz + + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Cache Spark + uses: actions/cache@v2 + with: + path: ~/.cache/spark + key: ${{ runner.os }}-spark + restore-keys: | + ${{ runner.os }}-spark- + + - name: Codecov + if: success() + run: | + codecov From 176e109b3e477e1295a060a6ee4957dcf3f381a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 7 Jun 2023 20:21:56 +0200 Subject: [PATCH 02/19] Update master.yml --- .github/workflows/master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 6e87551..66d6fef 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6] + python-version: [3.8, 3.11] steps: - uses: actions/checkout@v2 From 21da9eff1a3a929052a28e652f8d7547d80dcea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 7 Jun 2023 21:02:53 +0200 Subject: [PATCH 03/19] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 122cf20..3a4f8a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ipykernel -jedi<0.11 +jedi metakernel spylon[spark] tornado From e96e24495c3bdee464052dbe01a740d0dbee279e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 7 Jun 2023 21:09:12 +0200 Subject: [PATCH 04/19] Update init_spark_magic.py --- spylon_kernel/init_spark_magic.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spylon_kernel/init_spark_magic.py b/spylon_kernel/init_spark_magic.py index dd10300..09253d1 100644 --- a/spylon_kernel/init_spark_magic.py +++ b/spylon_kernel/init_spark_magic.py @@ -8,7 +8,6 @@ try: import jedi from jedi.api.helpers import get_on_completion_name - from jedi import common except ImportError as ex: jedi = None @@ -79,7 +78,7 @@ def get_completions(self, info): position = (info['line_num'], info['column']) interpreter = jedi.Interpreter(text, [self.env]) - lines = common.splitlines(text) + lines = text.splitlines() name = get_on_completion_name( interpreter._get_module_node(), lines, @@ -89,4 +88,4 @@ def get_completions(self, info): before = text[:len(text) - len(name)] completions = interpreter.completions() completions = [before + c.name_with_symbols for c in completions] - return [c[info['start']:] for c in completions] \ No newline at end of file + return [c[info['start']:] for c in completions] From 6b2387dd7a042f387b6bf923884edaf967b34d38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 7 Jun 2023 21:16:11 +0200 Subject: [PATCH 05/19] Update test_scala_interpreter.py --- test/test_scala_interpreter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_scala_interpreter.py b/test/test_scala_interpreter.py index 163b62b..caa4d53 100644 --- a/test/test_scala_interpreter.py +++ b/test/test_scala_interpreter.py @@ -11,7 +11,7 @@ def scala_interpreter(request): def test_simple_expression(scala_interpreter): result = scala_interpreter.interpret("4 + 4") - assert re.match('res\d+: Int = 8\n', result) + assert re.match(r'res\d+: Int = 8\n', result) def test_completion(scala_interpreter): @@ -94,4 +94,4 @@ def test_case_classes(scala_interpreter): val res = df.agg(sum('y)).collect().head''') strres = scala_interpreter.interpret("res.getLong(0)") result = scala_interpreter.last_result() - assert result == sum(range(10)) \ No newline at end of file + assert result == sum(range(10)) From fa13743dfdc861e84f69eff0978b08a8da5236c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 7 Jun 2023 21:23:29 +0200 Subject: [PATCH 06/19] Update test_scala_kernel.py --- test/test_scala_kernel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_scala_kernel.py b/test/test_scala_kernel.py index 28a574a..10b10f2 100644 --- a/test/test_scala_kernel.py +++ b/test/test_scala_kernel.py @@ -42,7 +42,7 @@ def test_simple_expression(spylon_kernel): result = spylon_kernel.do_execute_direct("4 + 4") assert isinstance(result, TextOutput) output = result.output - assert re.match('res\d+: Int = 8\n', output) + assert re.match(r'res\d+: Int = 8\n', output) def test_exception(spylon_kernel): @@ -109,4 +109,4 @@ def test_stdout(spylon_kernel): // Sleep for a bit since the process for getting text output is asynchronous Thread.sleep(1000)''') writes, _ = spylon_kernel.Writes.pop() - assert writes[0].strip() == 'test_stdout' \ No newline at end of file + assert writes[0].strip() == 'test_stdout' From 68f4ae909f712279259aa862d9bf43784dfcc190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 7 Jun 2023 21:50:11 +0200 Subject: [PATCH 07/19] Update test_spylon_kernel_jkt.py --- test_spylon_kernel_jkt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_spylon_kernel_jkt.py b/test_spylon_kernel_jkt.py index 4f2f70e..035df24 100644 --- a/test_spylon_kernel_jkt.py +++ b/test_spylon_kernel_jkt.py @@ -41,7 +41,7 @@ class SpylonKernelTests(jupyter_kernel_test.KernelTests): 'result': 'x: Int = 1\n' }, { 'code': 'val y = 1 to 3', - 'result': 'y: scala.collection.immutable.Range.Inclusive = Range(1, 2, 3)\n' + 'result': 'y: scala.collection.immutable.Range.Inclusive = Range 1 to 3\n' }] spark_configured = False @@ -56,4 +56,4 @@ def setUp(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 427c79962fbe9c2d910a1e5bd1d9110c54cc3a40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 7 Jun 2023 21:55:21 +0200 Subject: [PATCH 08/19] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e50b296..8bdd7c9 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ url='http://github.com/maxpoint/spylon-kernel', install_requires=[ 'ipykernel', - 'jedi>=0.10', + 'jedi', 'metakernel', 'spylon[spark]', 'tornado', From f88431f84805e9a79854d9d8769107ea59f8c622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 7 Jun 2023 22:02:54 +0200 Subject: [PATCH 09/19] Update scala_kernel.py --- spylon_kernel/scala_kernel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spylon_kernel/scala_kernel.py b/spylon_kernel/scala_kernel.py index 5f94833..d0052f4 100644 --- a/spylon_kernel/scala_kernel.py +++ b/spylon_kernel/scala_kernel.py @@ -18,7 +18,7 @@ class SpylonKernel(MetaKernel): implementation = 'spylon-kernel' implementation_version = get_versions()['version'] language = 'scala' - language_version = '2.11' + language_version = '2.12' banner = "spylon-kernel - evaluates Scala statements and expressions." language_info = { 'mimetype': 'text/x-scala', From 4236fc194999ad3491d6018df0d2f2126e705ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Wed, 7 Jun 2023 22:27:56 +0200 Subject: [PATCH 10/19] Update scala_interpreter.py --- spylon_kernel/scala_interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spylon_kernel/scala_interpreter.py b/spylon_kernel/scala_interpreter.py index 789a56a..9c580fc 100644 --- a/spylon_kernel/scala_interpreter.py +++ b/spylon_kernel/scala_interpreter.py @@ -52,7 +52,7 @@ def init_spark(conf=None, capture_stderr=False): # Create a temp directory that gets cleaned up on exit output_dir = os.path.abspath(tempfile.mkdtemp()) - def cleanup(): + def cleanup(signalnum=None, frame=None): shutil.rmtree(output_dir, True) atexit.register(cleanup) signal.signal(signal.SIGTERM, cleanup) From 691ffe7c6c9dec4ee9861d065bb9c510059981ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Thu, 8 Jun 2023 14:40:17 +0200 Subject: [PATCH 11/19] Update master.yml --- .github/workflows/master.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 66d6fef..3edacc7 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -30,12 +30,16 @@ jobs: pip install -r requirements-test.txt pip install codecov python -m spylon_kernel install --user + env: + JUPYTER_PLATFORM_DIRS: 1 - name: Test with pytest run: | python run_tests.py -vxrs --capture=sys --color=yes python setup.py sdist pip install --no-binary :all: dist/*.tar.gz + env: + JUPYTER_PLATFORM_DIRS: 1 - name: Cache pip uses: actions/cache@v2 @@ -57,3 +61,4 @@ jobs: if: success() run: | codecov + From 6b35ac0bee50d501ec219e96a9bc318e0cea9f8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Thu, 8 Jun 2023 15:12:38 +0200 Subject: [PATCH 12/19] Update scala_interpreter.py --- spylon_kernel/scala_interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spylon_kernel/scala_interpreter.py b/spylon_kernel/scala_interpreter.py index 9c580fc..cc0e9ae 100644 --- a/spylon_kernel/scala_interpreter.py +++ b/spylon_kernel/scala_interpreter.py @@ -408,7 +408,7 @@ def interpret(self, code): """ # Ensure the cell is not incomplete. Same approach taken by Apache Zeppelin. # https://github.com/apache/zeppelin/blob/3219218620e795769e6f65287f134b6a43e9c010/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java#L1263 - code = 'print("")\n'+code + code = code if code.endswith('\n') else code+'\n' try: res = self.jimain.interpret(code, False) From b25a6a4fb570a1fa02d4f12fae5167ce0af318c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Thu, 8 Jun 2023 17:45:27 +0200 Subject: [PATCH 13/19] Update test_scala_kernel.py --- test/test_scala_kernel.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_scala_kernel.py b/test/test_scala_kernel.py index 10b10f2..9d56026 100644 --- a/test/test_scala_kernel.py +++ b/test/test_scala_kernel.py @@ -71,11 +71,15 @@ def test_iscomplete(spylon_kernel): def test_last_result(spylon_kernel): - spylon_kernel.do_execute_direct(""" + result = spylon_kernel.do_execute_direct(""" case class LastResult(member: Int) val foo = LastResult(8) """) + print("Execution result: ", result) + foo = spylon_kernel.get_variable("foo") + print("Retrieved foo: ", foo) + assert foo From 810d58aa41feb6766b6fd25e5f2c5fb5bf74c092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Thu, 8 Jun 2023 18:37:01 +0200 Subject: [PATCH 14/19] Update scala_interpreter.py --- spylon_kernel/scala_interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spylon_kernel/scala_interpreter.py b/spylon_kernel/scala_interpreter.py index cc0e9ae..9c580fc 100644 --- a/spylon_kernel/scala_interpreter.py +++ b/spylon_kernel/scala_interpreter.py @@ -408,7 +408,7 @@ def interpret(self, code): """ # Ensure the cell is not incomplete. Same approach taken by Apache Zeppelin. # https://github.com/apache/zeppelin/blob/3219218620e795769e6f65287f134b6a43e9c010/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java#L1263 - code = code if code.endswith('\n') else code+'\n' + code = 'print("")\n'+code try: res = self.jimain.interpret(code, False) From 7668fda9f7a1526d06bedc7dcc7a4f6259e985b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Thu, 8 Jun 2023 20:16:06 +0200 Subject: [PATCH 15/19] add sonar to gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f98260d..fb0f2f6 100644 --- a/.gitignore +++ b/.gitignore @@ -95,4 +95,7 @@ ENV/ README.rst # Extra development notebooks -Untitled*.ipynb \ No newline at end of file +Untitled*.ipynb + +# sonar +.scannerwork \ No newline at end of file From 6672e19dd25139e80e9f3a3840aa6769b0bfd86b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Sat, 10 Jun 2023 14:08:55 +0200 Subject: [PATCH 16/19] from parso import split_lines --- spylon_kernel/init_spark_magic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spylon_kernel/init_spark_magic.py b/spylon_kernel/init_spark_magic.py index 09253d1..a58fae0 100644 --- a/spylon_kernel/init_spark_magic.py +++ b/spylon_kernel/init_spark_magic.py @@ -2,6 +2,7 @@ import logging import spylon.spark +from parso import split_lines from metakernel import Magic, option from .scala_interpreter import init_spark @@ -78,7 +79,7 @@ def get_completions(self, info): position = (info['line_num'], info['column']) interpreter = jedi.Interpreter(text, [self.env]) - lines = text.splitlines() + lines = split_lines(text) name = get_on_completion_name( interpreter._get_module_node(), lines, From 8ea86cdd139ea3a6bc60faa47134e827d6204608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Sun, 11 Jun 2023 18:39:23 +0200 Subject: [PATCH 17/19] undo some prints --- spylon_kernel/scala_interpreter.py | 2 +- test/test_scala_interpreter.py | 4 ++-- test/test_scala_kernel.py | 10 +++------- test_spylon_kernel_jkt.py | 2 +- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/spylon_kernel/scala_interpreter.py b/spylon_kernel/scala_interpreter.py index 9c580fc..789a56a 100644 --- a/spylon_kernel/scala_interpreter.py +++ b/spylon_kernel/scala_interpreter.py @@ -52,7 +52,7 @@ def init_spark(conf=None, capture_stderr=False): # Create a temp directory that gets cleaned up on exit output_dir = os.path.abspath(tempfile.mkdtemp()) - def cleanup(signalnum=None, frame=None): + def cleanup(): shutil.rmtree(output_dir, True) atexit.register(cleanup) signal.signal(signal.SIGTERM, cleanup) diff --git a/test/test_scala_interpreter.py b/test/test_scala_interpreter.py index caa4d53..163b62b 100644 --- a/test/test_scala_interpreter.py +++ b/test/test_scala_interpreter.py @@ -11,7 +11,7 @@ def scala_interpreter(request): def test_simple_expression(scala_interpreter): result = scala_interpreter.interpret("4 + 4") - assert re.match(r'res\d+: Int = 8\n', result) + assert re.match('res\d+: Int = 8\n', result) def test_completion(scala_interpreter): @@ -94,4 +94,4 @@ def test_case_classes(scala_interpreter): val res = df.agg(sum('y)).collect().head''') strres = scala_interpreter.interpret("res.getLong(0)") result = scala_interpreter.last_result() - assert result == sum(range(10)) + assert result == sum(range(10)) \ No newline at end of file diff --git a/test/test_scala_kernel.py b/test/test_scala_kernel.py index 9d56026..28a574a 100644 --- a/test/test_scala_kernel.py +++ b/test/test_scala_kernel.py @@ -42,7 +42,7 @@ def test_simple_expression(spylon_kernel): result = spylon_kernel.do_execute_direct("4 + 4") assert isinstance(result, TextOutput) output = result.output - assert re.match(r'res\d+: Int = 8\n', output) + assert re.match('res\d+: Int = 8\n', output) def test_exception(spylon_kernel): @@ -71,15 +71,11 @@ def test_iscomplete(spylon_kernel): def test_last_result(spylon_kernel): - result = spylon_kernel.do_execute_direct(""" + spylon_kernel.do_execute_direct(""" case class LastResult(member: Int) val foo = LastResult(8) """) - print("Execution result: ", result) - foo = spylon_kernel.get_variable("foo") - print("Retrieved foo: ", foo) - assert foo @@ -113,4 +109,4 @@ def test_stdout(spylon_kernel): // Sleep for a bit since the process for getting text output is asynchronous Thread.sleep(1000)''') writes, _ = spylon_kernel.Writes.pop() - assert writes[0].strip() == 'test_stdout' + assert writes[0].strip() == 'test_stdout' \ No newline at end of file diff --git a/test_spylon_kernel_jkt.py b/test_spylon_kernel_jkt.py index 035df24..204a1ea 100644 --- a/test_spylon_kernel_jkt.py +++ b/test_spylon_kernel_jkt.py @@ -56,4 +56,4 @@ def setUp(self): if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file From 92a045952244a1eaf139621d7251ef4be42071df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Sun, 11 Jun 2023 18:59:18 +0200 Subject: [PATCH 18/19] add re.match(r' --- test/test_scala_interpreter.py | 2 +- test/test_scala_kernel.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_scala_interpreter.py b/test/test_scala_interpreter.py index 163b62b..f14f27e 100644 --- a/test/test_scala_interpreter.py +++ b/test/test_scala_interpreter.py @@ -11,7 +11,7 @@ def scala_interpreter(request): def test_simple_expression(scala_interpreter): result = scala_interpreter.interpret("4 + 4") - assert re.match('res\d+: Int = 8\n', result) + assert re.match(r'res\d+: Int = 8\n', result) def test_completion(scala_interpreter): diff --git a/test/test_scala_kernel.py b/test/test_scala_kernel.py index 28a574a..43e3a95 100644 --- a/test/test_scala_kernel.py +++ b/test/test_scala_kernel.py @@ -42,7 +42,7 @@ def test_simple_expression(spylon_kernel): result = spylon_kernel.do_execute_direct("4 + 4") assert isinstance(result, TextOutput) output = result.output - assert re.match('res\d+: Int = 8\n', output) + assert re.match(r'res\d+: Int = 8\n', output) def test_exception(spylon_kernel): From 7784f9ca2ecd46db468555709e468680d3048a49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20J=C3=B8rgensen?= Date: Sun, 11 Jun 2023 19:15:32 +0200 Subject: [PATCH 19/19] def cleanup(signalnum=None, frame=None) --- spylon_kernel/scala_interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spylon_kernel/scala_interpreter.py b/spylon_kernel/scala_interpreter.py index 789a56a..9c580fc 100644 --- a/spylon_kernel/scala_interpreter.py +++ b/spylon_kernel/scala_interpreter.py @@ -52,7 +52,7 @@ def init_spark(conf=None, capture_stderr=False): # Create a temp directory that gets cleaned up on exit output_dir = os.path.abspath(tempfile.mkdtemp()) - def cleanup(): + def cleanup(signalnum=None, frame=None): shutil.rmtree(output_dir, True) atexit.register(cleanup) signal.signal(signal.SIGTERM, cleanup)