Populate more fields for canvas courses (#2404)

* adding start date

* adding end  date

* adding method to get context info

* add url field

* adding migration to fix model

* adding migration to fix model

* fixing tests

* fixing migration order

* fixing migration order

* reverting model changes

* fixing issue with end dates missing

* add log

* log only if dates are undefined

* log only if dates are undefined

Author: shanbady

learning_resources/etl/canvas.py

@@ -4,6 +4,7 @@
 import zipfile
 from collections import defaultdict
 from collections.abc import Generator
+from datetime import datetime
 from io import BytesIO
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -73,20 +74,40 @@ def sync_canvas_archive(bucket, key: str, overwrite):
     return resource_readable_id, run
 
 
+def _course_url(course_archive_path) -> str:
+    context_info = parse_context_xml(course_archive_path)
+    return f"https://{context_info.get('canvas_domain')}/courses/{context_info.get('course_id')}/"
+
+
 def run_for_canvas_archive(course_archive_path, course_folder, overwrite):
     """
     Generate and return a LearningResourceRun for a Canvas course
     """
     checksum = calc_checksum(course_archive_path)
     course_info = parse_canvas_settings(course_archive_path)
     course_title = course_info.get("title")
+    url = _course_url(course_archive_path)
+    start_at = course_info.get("start_at")
+    end_at = course_info.get("conclude_at")
+    if start_at:
+        try:
+            start_at = datetime.fromisoformat(start_at)
+        except (ValueError, TypeError):
+            log.warning("Invalid start_at date format: %s", start_at)
+    if end_at:
+        try:
+            end_at = datetime.fromisoformat(end_at)
+        except (ValueError, TypeError):
+            log.warning("Invalid start_at date format: %s", end_at)
+
     readable_id = f"{course_folder}-{course_info.get('course_code')}"
     # create placeholder learning resource
     resource, _ = LearningResource.objects.update_or_create(
         readable_id=readable_id,
         defaults={
             "title": course_title,
             "published": False,
+            "url": url,
             "test_mode": True,
             "etl_source": ETLSource.canvas.name,
             "platform": LearningResourcePlatform.objects.get(
@@ -100,6 +121,8 @@ def run_for_canvas_archive(course_archive_path, course_folder, overwrite):
             run_id=f"{readable_id}+canvas",
             learning_resource=resource,
             published=True,
+            start_date=start_at,
+            end_date=end_at,
         )
     run = resource.runs.first()
     resource_readable_id = run.learning_resource.readable_id
@@ -206,6 +229,21 @@ def transform_canvas_problem_files(
             yield problem_file_data
 
 
+def parse_context_xml(course_archive_path: str) -> dict:
+    with zipfile.ZipFile(course_archive_path, "r") as course_archive:
+        context = course_archive.read("course_settings/context.xml")
+    root = ElementTree.fromstring(context)
+    namespaces = {"ns": "http://canvas.instructure.com/xsd/cccv1p0"}
+    context_info = {}
+    item_keys = ["course_id", "root_account_id", "canvas_domain", "root_account_name"]
+    for key in item_keys:
+        element = root.find(f"ns:{key}", namespaces)
+        if element is not None:
+            context_info[key] = element.text
+
+    return context_info
+
+
 def parse_module_meta(course_archive_path: str) -> dict:
     """
     Parse module_meta.xml and return publish/active status of resources.

learning_resources/etl/canvas_test.py

@@ -111,12 +111,17 @@ def test_run_for_canvas_archive_creates_resource_and_run(tmp_path, mocker):
         "learning_resources.etl.canvas.parse_canvas_settings",
         return_value={"title": "Test Course", "course_code": "TEST101"},
     )
+    mocker.patch(
+        "learning_resources.etl.canvas.parse_context_xml",
+        return_value={"course_id": "123", "canvas_domain": "mit.edu"},
+    )
+
     mocker.patch("learning_resources.etl.canvas.calc_checksum", return_value="abc123")
     # No resource exists yet
-    course_archive_path = tmp_path / "archive.zip"
-    course_archive_path.write_text("dummy")
+    zip_path = tmp_path / "archive.zip"
+
     _, run = run_for_canvas_archive(
-        course_archive_path, course_folder=course_folder, overwrite=True
+        zip_path, course_folder=course_folder, overwrite=True
     )
     resource = LearningResource.objects.get(readable_id=f"{course_folder}-TEST101")
     assert resource.title == "Test Course"
@@ -138,6 +143,10 @@ def test_run_for_canvas_archive_creates_run_if_none_exists(tmp_path, mocker):
         "learning_resources.etl.canvas.parse_canvas_settings",
         return_value={"title": "Test Course", "course_code": "TEST104"},
     )
+    mocker.patch(
+        "learning_resources.etl.canvas.parse_context_xml",
+        return_value={"course_id": "123", "canvas_domain": "mit.edu"},
+    )
     mocker.patch(
         "learning_resources.etl.canvas.calc_checksum", return_value="checksum104"
     )