Lifecycle identity refactor (#428)

* Baseline refactor, tests still failing

* Don't overwrite with v0.0 if set

* More refactoring, tests still failing

* Wrong mapping

* Move to serialization phase

* Preserve existing cell ID semantics

* Make linter happy

* Fmt

* None until v2

* Deserialize options only

examples/grpc-client/main.go

@@ -73,6 +73,9 @@ func run() error {
 
 		resp, err := client.Deserialize(context.Background(), &parserv1.DeserializeRequest{
 			Source: data,
+			Options: &parserv1.DeserializeRequestOptions{
+				Identity: parserv1.RunmeIdentity_RUNME_IDENTITY_UNSPECIFIED,
+			},
 		})
 		if err != nil {
 			return err
@@ -85,9 +88,6 @@ func run() error {
 
 		resp2, err := client.Serialize(context.Background(), &parserv1.SerializeRequest{
 			Notebook: resp.Notebook,
-			Options: &parserv1.SerializeRequestOptions{
-				Identity: parserv1.RunmeIdentity_RUNME_IDENTITY_UNSPECIFIED,
-			},
 		})
 		if err != nil {
 			return err

internal/api/runme/parser/v1/parser.proto

@@ -55,13 +55,8 @@ message DeserializeResponse {
   Notebook notebook = 1;
 }
 
-message SerializeRequestOptions {
-  RunmeIdentity identity = 1;
-}
-
 message SerializeRequest {
   Notebook notebook = 1;
-  SerializeRequestOptions options = 2;
 }
 
 message SerializeResponse {

internal/document/block.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/stateful/runme/internal/document/identity"
 	"github.com/stateful/runme/internal/executable"
 	"github.com/stateful/runme/internal/shell"
 	"github.com/yuin/goldmark/ast"
@@ -50,6 +51,8 @@ func (b CodeBlocks) Names() (result []string) {
 type Renderer func(ast.Node, []byte) ([]byte, error)
 
 type CodeBlock struct {
+	id            string
+	idGenerated   bool
 	attributes    map[string]string
 	inner         *ast.FencedCodeBlock
 	intro         string
@@ -62,6 +65,7 @@ type CodeBlock struct {
 
 func newCodeBlock(
 	node *ast.FencedCodeBlock,
+	identityResolver *identity.IdentityResolver,
 	nameResolver *nameResolver,
 	source []byte,
 	render Renderer,
@@ -71,6 +75,8 @@ func newCodeBlock(
 		return nil, err
 	}
 
+	id, hasID := identityResolver.GetCellID(node, attributes)
+
 	name, hasName := getName(node, source, nameResolver, attributes)
 
 	value, err := render(node, source)
@@ -79,6 +85,8 @@ func newCodeBlock(
 	}
 
 	return &CodeBlock{
+		id:            id,
+		idGenerated:   !hasID,
 		attributes:    attributes,
 		inner:         node,
 		intro:         getIntro(node, source),
@@ -100,6 +108,8 @@ func (b *CodeBlock) Clone() *CodeBlock {
 		attributes[key] = value
 	}
 	return &CodeBlock{
+		id:            b.id,
+		idGenerated:   b.idGenerated,
 		attributes:    attributes,
 		intro:         b.intro,
 		language:      b.language,
@@ -149,6 +159,10 @@ func (b *CodeBlock) Lines() []string {
 	return b.lines
 }
 
+func (b *CodeBlock) ID() string {
+	return b.id
+}
+
 func (b *CodeBlock) Name() string {
 	return b.name
 }

internal/document/document.go

@@ -11,19 +11,22 @@ import (
 	"github.com/yuin/goldmark/text"
 
 	"github.com/stateful/runme/internal/document/constants"
+	"github.com/stateful/runme/internal/document/identity"
 )
 
 type Document struct {
-	astNode      ast.Node
-	nameResolver *nameResolver
-	node         *Node
-	parser       parser.Parser
-	renderer     Renderer
-	source       []byte
+	astNode          ast.Node
+	identityResolver *identity.IdentityResolver
+	nameResolver     *nameResolver
+	node             *Node
+	parser           parser.Parser
+	renderer         Renderer
+	source           []byte
 }
 
-func New(source []byte, renderer Renderer) *Document {
+func New(source []byte, renderer Renderer, identityResolver *identity.IdentityResolver) *Document {
 	return &Document{
+		identityResolver: identityResolver,
 		nameResolver: &nameResolver{
 			namesCounter: map[string]int{},
 			cache:        map[interface{}]string{},
@@ -63,6 +66,7 @@ func (d *Document) buildBlocksTree(parent ast.Node, node *Node) error {
 		case ast.KindFencedCodeBlock:
 			block, err := newCodeBlock(
 				astNode.(*ast.FencedCodeBlock),
+				d.identityResolver,
 				d.nameResolver,
 				d.source,
 				d.renderer,

internal/document/document_test.go

@@ -4,11 +4,14 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/stateful/runme/internal/document/identity"
 	"github.com/stateful/runme/internal/renderer/cmark"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+var identityResolver = identity.NewResolver(identity.AllLifecycleIdentity)
+
 func TestDocument_Parse(t *testing.T) {
 	data := []byte(`# Examples
 
@@ -32,7 +35,7 @@ First paragraph.
 2. Item 2
 3. Item 3
 `)
-	doc := New(data, cmark.Render)
+	doc := New(data, cmark.Render, identityResolver)
 	node, _, err := doc.Parse()
 	require.NoError(t, err)
 	assert.Len(t, node.children, 4)
@@ -84,7 +87,7 @@ console.log("hello world!")
 `,
 	))
 
-	doc := New(data, cmark.Render)
+	doc := New(data, cmark.Render, identityResolver)
 	node, _, err := doc.Parse()
 	require.NoError(t, err)
 
@@ -98,7 +101,7 @@ func TestDocument_FinalLineBreaks(t *testing.T) {
 	data := []byte(`This will test final line breaks`)
 
 	t.Run("No breaks", func(t *testing.T) {
-		doc := New(data, cmark.Render)
+		doc := New(data, cmark.Render, identityResolver)
 		_, astNode, err := doc.Parse()
 		require.NoError(t, err)
 
@@ -114,7 +117,7 @@ func TestDocument_FinalLineBreaks(t *testing.T) {
 
 	t.Run("1 LF", func(t *testing.T) {
 		withLineBreaks := append(data, bytes.Repeat([]byte{'\n'}, 1)...)
-		doc := New(withLineBreaks, cmark.Render)
+		doc := New(withLineBreaks, cmark.Render, identityResolver)
 		_, astNode, err := doc.Parse()
 		require.NoError(t, err)
 
@@ -130,7 +133,7 @@ func TestDocument_FinalLineBreaks(t *testing.T) {
 
 	t.Run("1 CRLF", func(t *testing.T) {
 		withLineBreaks := append(data, bytes.Repeat([]byte{'\r', '\n'}, 1)...)
-		doc := New(withLineBreaks, cmark.Render)
+		doc := New(withLineBreaks, cmark.Render, identityResolver)
 		_, astNode, err := doc.Parse()
 		require.NoError(t, err)
 
@@ -146,7 +149,7 @@ func TestDocument_FinalLineBreaks(t *testing.T) {
 
 	t.Run("7 LFs", func(t *testing.T) {
 		withLineBreaks := append(data, bytes.Repeat([]byte{'\n'}, 7)...)
-		doc := New(withLineBreaks, cmark.Render)
+		doc := New(withLineBreaks, cmark.Render, identityResolver)
 		_, astNode, err := doc.Parse()
 		require.NoError(t, err)
 

internal/document/editor/cell.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 
 	"github.com/stateful/runme/internal/document"
-	"github.com/stateful/runme/internal/identity"
+	ulid "github.com/stateful/runme/internal/ulid"
 	"github.com/yuin/goldmark/ast"
 )
 
@@ -54,13 +54,15 @@ type Notebook struct {
 	frontmatterParseInfo *document.FrontmatterParseInfo
 }
 
-func (c *Cell) EnsureID() {
-	id, ok := c.Metadata["id"]
-	if ok && identity.ValidID(id) {
-		return
+// This mimics what otherwise would happen in the extension
+func (n *Notebook) ForceLifecyleIdentities() {
+	for _, c := range n.Cells {
+		id, ok := c.Metadata[PrefixAttributeName(InternalAttributePrefix, "id")]
+		if !ok && id == "" || !ulid.ValidID(id) {
+			continue
+		}
+		c.Metadata["id"] = id
 	}
-
-	c.Metadata["id"] = identity.GenerateID()
 }
 
 func (n *Notebook) GetContentOffset() int {
@@ -147,6 +149,10 @@ func toCellsRec(
 
 			// In the future, we will include language detection (#77).
 			metadata := block.Attributes()
+			cellID := block.ID()
+			if cellID != "" {
+				metadata[PrefixAttributeName(InternalAttributePrefix, "id")] = cellID
+			}
 			metadata[PrefixAttributeName(InternalAttributePrefix, "name")] = block.Name()
 			*cells = append(*cells, &Cell{
 				Kind:       CodeKind,

internal/document/editor/cell_test.go

@@ -99,7 +99,7 @@ Test paragraph
 )
 
 func Test_toCells_DataNested(t *testing.T) {
-	doc := document.New(testDataNested, cmark.Render)
+	doc := document.New(testDataNested, cmark.Render, identityResolverAll)
 	node, _, err := doc.Parse()
 	require.NoError(t, err)
 	cells := toCells(node, testDataNested)
@@ -122,7 +122,7 @@ func Test_toCells_Lists(t *testing.T) {
 2. Item 2
 3. Item 3
 `)
-		doc := document.New(data, cmark.Render)
+		doc := document.New(data, cmark.Render, identityResolverAll)
 		node, _, err := doc.Parse()
 		require.NoError(t, err)
 		cells := toCells(node, data)
@@ -138,7 +138,7 @@ func Test_toCells_Lists(t *testing.T) {
    ` + "```" + `
 3. Item 3
 `)
-		doc := document.New(data, cmark.Render)
+		doc := document.New(data, cmark.Render, identityResolverAll)
 		node, _, err := doc.Parse()
 		require.NoError(t, err)
 		cells := toCells(node, data)
@@ -155,7 +155,7 @@ func Test_toCells_EmptyLang(t *testing.T) {
 echo 1
 ` + "```" + `
 `)
-	doc := document.New(data, cmark.Render)
+	doc := document.New(data, cmark.Render, identityResolverAll)
 	node, _, err := doc.Parse()
 	require.NoError(t, err)
 	cells := toCells(node, data)
@@ -173,7 +173,7 @@ def hello():
     print("Hello World")
 ` + "```" + `
 `)
-	doc := document.New(data, cmark.Render)
+	doc := document.New(data, cmark.Render, identityResolverAll)
 	node, _, err := doc.Parse()
 	require.NoError(t, err)
 	cells := toCells(node, data)
@@ -196,7 +196,7 @@ Last paragraph.
 `)
 
 	parse := func() []*Cell {
-		doc := document.New(data, cmark.Render)
+		doc := document.New(data, cmark.Render, identityResolverAll)
 		node, _, err := doc.Parse()
 		require.NoError(t, err)
 		cells := toCells(node, data)
@@ -299,7 +299,7 @@ func Test_serializeCells_nestedCode(t *testing.T) {
    pre-commit install
    ` + "```" + `
 `)
-	doc := document.New(data, cmark.Render)
+	doc := document.New(data, cmark.Render, identityResolverAll)
 	node, _, err := doc.Parse()
 	require.NoError(t, err)
 	cells := toCells(node, data)
@@ -331,7 +331,7 @@ func Test_serializeCells(t *testing.T) {
 	t.Run("attributes_babikml", func(t *testing.T) {
 		data := []byte("```sh { name=echo first= second=2 }\necho 1\n```\n")
 		expected := []byte("```sh {\"first\":\"\",\"name\":\"echo\",\"second\":\"2\"}\necho 1\n```\n")
-		doc := document.New(data, cmark.Render)
+		doc := document.New(data, cmark.Render, identityResolverAll)
 		node, _, err := doc.Parse()
 		require.NoError(t, err)
 		cells := toCells(node, data)
@@ -340,7 +340,7 @@ func Test_serializeCells(t *testing.T) {
 
 	t.Run("attributes", func(t *testing.T) {
 		data := []byte("```sh {\"first\":\"\",\"name\":\"echo\",\"second\":\"2\"}\necho 1\n```\n")
-		doc := document.New(data, cmark.Render)
+		doc := document.New(data, cmark.Render, identityResolverAll)
 		node, _, err := doc.Parse()
 		require.NoError(t, err)
 		cells := toCells(node, data)
@@ -349,7 +349,7 @@ func Test_serializeCells(t *testing.T) {
 
 	t.Run("privateFields", func(t *testing.T) {
 		data := []byte("```sh {\"first\":\"\",\"name\":\"echo\",\"second\":\"2\"}\necho 1\n```\n")
-		doc := document.New(data, cmark.Render)
+		doc := document.New(data, cmark.Render, identityResolverAll)
 		node, _, err := doc.Parse()
 		require.NoError(t, err)
 
@@ -369,7 +369,7 @@ def hello():
 	print("Hello World")
 ` + "```" + `
 `)
-		doc := document.New(data, cmark.Render)
+		doc := document.New(data, cmark.Render, identityResolverAll)
 		node, _, err := doc.Parse()
 		require.NoError(t, err)
 		cells := toCells(node, data)
@@ -444,7 +444,7 @@ func Test_notebook_frontmatter(t *testing.T) {
 		t.Run(ex.kind, func(t *testing.T) {
 			t.Parallel()
 
-			notebook, err := Deserialize(file, false)
+			notebook, err := Deserialize(file, identityResolverAll)
 			require.NoError(t, err)
 
 			fmtr, info := notebook.ParsedFrontmatter()

internal/document/editor/editor.go

@@ -10,18 +10,19 @@ import (
 	"github.com/stateful/runme/internal/renderer/cmark"
 
 	"github.com/stateful/runme/internal/document/constants"
+	"github.com/stateful/runme/internal/document/identity"
 )
 
 const FrontmatterKey = "frontmatter"
 
-func Deserialize(data []byte, requireIdentity bool) (*Notebook, error) {
+func Deserialize(data []byte, identityResolver *identity.IdentityResolver) (*Notebook, error) {
 	sections, err := document.ParseSections(data)
 	if err != nil {
 		return nil, err
 	}
 
 	// Deserialize content to cells.
-	doc := document.New(sections.Content, cmark.Render)
+	doc := document.New(sections.Content, cmark.Render, identityResolver)
 	node, _, err := doc.Parse()
 	if err != nil {
 		return nil, err
@@ -37,7 +38,7 @@ func Deserialize(data []byte, requireIdentity bool) (*Notebook, error) {
 		PrefixAttributeName(InternalAttributePrefix, constants.FinalLineBreaksKey): fmt.Sprint(finalLinesBreaks),
 	}
 
-	f, info := document.ParseFrontmatterWithIdentity(string(sections.FrontMatter), requireIdentity)
+	f, info := document.ParseFrontmatterWithIdentity(string(sections.FrontMatter), identityResolver.DocumentEnabled())
 	notebook.parsedFrontmatter = &f
 	notebook.frontmatterParseInfo = &info