Migrate legacy notebooks in shared readers

aidialog/_modidx.py

@@ -226,12 +226,16 @@
                                 'aidialog.ipynb.Message.to_cell': ('ipynb.html#message.to_cell', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb._clean_out_meta': ('ipynb.html#_clean_out_meta', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb._output_from_cell': ('ipynb.html#_output_from_cell', 'aidialog/ipynb.py'),
+                                'aidialog.ipynb._v1_to_v2': ('ipynb.html#_v1_to_v2', 'aidialog/ipynb.py'),
+                                'aidialog.ipynb._v2_to_v3': ('ipynb.html#_v2_to_v3', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb.att2dict': ('ipynb.html#att2dict', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb.conv_old_prompts': ('ipynb.html#conv_old_prompts', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb.dict2att': ('ipynb.html#dict2att', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb.get_ipynb': ('ipynb.html#get_ipynb', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb.home_atts': ('ipynb.html#home_atts', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb.ipynb_cells': ('ipynb.html#ipynb_cells', 'aidialog/ipynb.py'),
+                                'aidialog.ipynb.migrate_nb': ('ipynb.html#migrate_nb', 'aidialog/ipynb.py'),
+                                'aidialog.ipynb.nb_ver': ('ipynb.html#nb_ver', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb.read_ipynb': ('ipynb.html#read_ipynb', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb.reads_ipynb': ('ipynb.html#reads_ipynb', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb.safe_mtime': ('ipynb.html#safe_mtime', 'aidialog/ipynb.py'),

aidialog/ipynb.py

@@ -5,8 +5,8 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_ipynb.ipynb.
 
 # %% auto #0
-__all__ = ['att2dict', 'home_atts', 'unhome_atts', 'get_ipynb', 'safe_mtime', 'write_ipynb', 'ipynb_cells', 'dict2att',
-           'reads_ipynb', 'read_ipynb', 'conv_old_prompts']
+__all__ = ['CUR_VER', 'att2dict', 'home_atts', 'unhome_atts', 'conv_old_prompts', 'nb_ver', 'migrate_nb', 'get_ipynb',
+           'safe_mtime', 'write_ipynb', 'ipynb_cells', 'dict2att', 'reads_ipynb', 'read_ipynb']
 
 # %% ../nbs/02_ipynb.ipynb #d3d0463b
 from fastcore.utils import *
@@ -16,6 +16,7 @@
 from contextlib import suppress
 from base64 import b64encode,b64decode
 from .dialog import *
+from .msg_parts import conv_tools
 
 # %% ../nbs/02_ipynb.ipynb #aaaeb85a
 def att2dict(att): return {att.content_type: b64encode(att.data).decode('ascii') if isinstance(att.data, bytes) else att.data}
@@ -59,29 +60,88 @@ def cell_meta(self:Message):
 _prompt_magic = '%%prompt'
 
 @patch
-def to_cell(self:Message, version=2):
+def to_cell(self:Message):
     "Convert message to a notebook cell"
     meta = self.cell_meta()
     src = self.source
     if self.msg_type==sprompt:
         meta['solveit_ai'] = True
         src = f'{_prompt_magic}\n{src}'
     outkw = {}
-    if self.msg_type in (scode,sprompt) and self.output:
-        outputs = self.output
-        if version==1 and self.msg_type==scode: outputs = json.loads(outputs)
-        outkw['outputs'] = [_clean_out_meta(o) for o in outputs]
+    if self.msg_type in (scode,sprompt) and self.output: outkw['outputs'] = [_clean_out_meta(o) for o in self.output]
     atts = {att.id: att2dict(att) for att in (self.attachments or [])}
     if atts: outkw['attachments'] = atts
     cell = mk_cell(src, self.cell_type, id=self.id, metadata=meta, **outkw)
     if repairs := repair_cell(cell): print('NB repair:', '; '.join(repairs))
     return cell
 
+# %% ../nbs/02_ipynb.ipynb #ded8e1b4
+_reply_sep = "\n\n##### 🤖Reply🤖<!-- SOLVEIT_SEPARATOR_7f3a9b2c -->\n\n"
+
+def conv_old_prompts(nb):
+    "Rewrite old markdown-form prompt cells to the code-cell form, in place; returns the changed cell ids"
+    changed = []
+    for c in nb['cells']:
+        if c['cell_type']!='markdown' or not c['metadata'].get('solveit_ai'): continue
+        if isinstance(c['source'], list): c['source'] = ''.join(c['source'])
+        content,*reply = c['source'].split(_reply_sep)
+        c['cell_type'] = 'code'
+        c['source'] = f'{_prompt_magic}\n{content}'
+        c['outputs'] = prompt_output(reply[0]) if reply else []
+        c['execution_count'] = None
+        changed.append(c['id'])
+    if changed: home_atts(nb)
+    return changed
+
+# %% ../nbs/02_ipynb.ipynb #migrations
+CUR_VER = 3
+
+def nb_ver(nb):
+    "Stored notebook version; unversioned files start at v1"
+    meta = nb.get('metadata', {})
+    return meta.get('solveit', {}).get('ver', meta.get('solveit_ver', 1))
+
+# %% ../nbs/02_ipynb.ipynb #migratev1
+def _v1_to_v2(nb):
+    meta = nb.setdefault('metadata', {})
+    sv = meta.get('solveit', {})
+    modes = {1:'standard', 2:'learning', 3:'concise'}
+    if isinstance(sv.get('mode'), int): sv['mode'] = modes[sv['mode']]
+    if isinstance(meta.get('solveit_dialog_mode'), int): meta['solveit_dialog_mode'] = modes[meta['solveit_dialog_mode']]
+
+# %% ../nbs/02_ipynb.ipynb #migratev2
+def _v2_to_v3(nb):
+    meta = nb.setdefault('metadata', {})
+    sv = meta.setdefault('solveit', {})
+    if 'solveit_dialog_mode' in meta: sv.setdefault('mode', meta.pop('solveit_dialog_mode'))
+    meta.pop('solveit_ver', None)
+    conv_old_prompts(nb)
+    for c in nb['cells']:
+        if not c['metadata'].get('solveit_ai'): continue
+        for o in c.get('outputs', []):
+            md = nested_idx(o, 'metadata', 'is_ai_res') and nested_idx(o, 'data', 'text/markdown')
+            if not md: continue
+            if '-usage-details' in md: o['data']['text/markdown'] = conv_tools(md)
+
+# %% ../nbs/02_ipynb.ipynb #migratenb
+_migrations = {1:_v1_to_v2, 2:_v2_to_v3}
+
+def migrate_nb(nb):
+    "Upgrade a normalized notebook in place. Return whether its version changed"
+    ver = nb_ver(nb)
+    if not 1 <= ver <= CUR_VER: raise ValueError(f'Unsupported notebook version: {ver}')
+    for v in range(ver, CUR_VER):
+        _migrations[v](nb)
+        nb.setdefault('metadata', {}).setdefault('solveit', {})['ver'] = v+1
+    return ver < CUR_VER
+
 # %% ../nbs/02_ipynb.ipynb #10a73cfc
-def get_ipynb(dlg:Dialog, version=2, msgs=None):
+def get_ipynb(dlg:Dialog, msgs=None):
     "Notebook object for `dlg`; `msgs` defaults to all its messages"
-    cells = [m.to_cell(version=version) for m in (dlg.messages if msgs is None else msgs)]
-    nb = new_nb(cells=cells, meta=dict(dlg.meta))
+    cells = [m.to_cell() for m in (dlg.messages if msgs is None else msgs)]
+    meta = dict(dlg.meta)
+    meta['solveit'] = dict(meta.get('solveit', {}), ver=CUR_VER)
+    nb = new_nb(cells=cells, meta=meta)
     home_atts(nb)
     if repairs := repair_nb(nb): print('NB repair:', '; '.join(repairs))
     return nb
@@ -94,18 +154,19 @@ def safe_mtime(p):
     with suppress(FileNotFoundError): return p.stat(follow_symlinks=False).st_mtime
 
 # %% ../nbs/02_ipynb.ipynb #ea5fa1dd
-def write_ipynb(dlg:Dialog, fname=None, version=2, msgs=None, **kwargs):
+def write_ipynb(dlg:Dialog, fname=None, msgs=None, **kwargs):
     "Write `dlg` as a notebook, or return the JSON string if `fname` is None; `kwargs` (e.g. `uid`/`gid`) pass to `atomic_save`"
-    res = nb2str(get_ipynb(dlg, version=version, msgs=msgs))
+    res = nb2str(get_ipynb(dlg, msgs=msgs))
     if not fname: return res
     fname = Path(fname).expanduser()
     with atomic_save(fname, mode='w', encoding='utf-8', **kwargs) as f: f.write(res)
     dlg.mtime_ = safe_mtime(fname)
+    dlg.migrated_ = False
 
 # %% ../nbs/02_ipynb.ipynb #5527e596
 @patch
-def write(self:Dialog, base_path, version=2, msgs=None, **kwargs):
-    write_ipynb(self, Path(base_path).expanduser()/f'{self.name}.ipynb', version=version, msgs=msgs, **kwargs)
+def write(self:Dialog, base_path, msgs=None, **kwargs):
+    write_ipynb(self, Path(base_path).expanduser()/f'{self.name}.ipynb', msgs=msgs, **kwargs)
 
 # %% ../nbs/02_ipynb.ipynb #0e16e93a
 def ipynb_cells(path, nm, prefix=None, suffix=None):
@@ -114,6 +175,8 @@ def ipynb_cells(path, nm, prefix=None, suffix=None):
     try: nb = read_nb(tmpl)
     except json.JSONDecodeError: return []
     if repairs := repair_nb(nb): print('NB repair:', '; '.join(repairs))
+    migrate_nb(nb)
+    unhome_atts(nb)
     return listify(prefix) + nb.cells + listify(suffix)
 
 # %% ../nbs/02_ipynb.ipynb #ba0bb61e
@@ -155,7 +218,11 @@ def reads_ipynb(txt, cls=Dialog, name='dialog', verbose=False):
     if (repairs := repair_nb(nb)) and verbose: print('NB repair:', '; '.join(repairs))
     unhome_atts(nb)
     nb = dict2nb(nb)
-    return cls(name=name, meta=dict(nb.get('metadata', {}))).from_cells(nb.cells)
+    migrated = migrate_nb(nb)
+    unhome_atts(nb)
+    res = cls(name=name, meta=dict(nb.get('metadata', {}))).from_cells(nb.cells)
+    res.migrated_ = migrated
+    return res
 
 # %% ../nbs/02_ipynb.ipynb #7e8912e8
 def read_ipynb(fname, cls=Dialog, name=None, verbose=False):
@@ -176,21 +243,3 @@ def save(self:Dialog, fname=None):
     fname = fname or self.path_
     if not fname: raise ValueError('no fname passed, and no `path_` stamped by read_ipynb')
     write_ipynb(self, fname)
-
-# %% ../nbs/02_ipynb.ipynb #ded8e1b4
-_reply_sep = "\n\n##### 🤖Reply🤖<!-- SOLVEIT_SEPARATOR_7f3a9b2c -->\n\n"
-
-def conv_old_prompts(nb):
-    "Rewrite old markdown-form prompt cells to the code-cell form, in place; returns the changed cell ids"
-    changed = []
-    for c in nb['cells']:
-        if c['cell_type']!='markdown' or not c['metadata'].get('solveit_ai'): continue
-        if isinstance(c['source'], list): c['source'] = ''.join(c['source'])
-        content,*reply = c['source'].split(_reply_sep)
-        c['cell_type'] = 'code'
-        c['source'] = f'{_prompt_magic}\n{content}'
-        c['outputs'] = prompt_output(reply[0]) if reply else []
-        c['execution_count'] = None
-        changed.append(c['id'])
-    if changed: home_atts(nb)
-    return changed