remove QueryModifiers, replace with explicit

db.go

@@ -130,11 +130,6 @@ func (tdb *TelemDb) AddEvents(events ...skylab.BusEvent) (int64, error) {
 	return tdb.AddEventsCtx(context.Background(), events...)
 }
 
-// QueryModifier augments SQL strings.
-type QueryModifier interface {
-	ModifyStatement(*strings.Builder) error
-}
-
 // LimitOffsetModifier is a modifier to support pagniation.
 type LimitOffsetModifier struct {
 	Limit  int
@@ -147,30 +142,24 @@ func (l *LimitOffsetModifier) ModifyStatement(sb *strings.Builder) error {
 	return nil
 }
 
-type OrderByTimestampModifer struct {
-}
-
-func (o *OrderByTimestampModifer) ModifyStatement(sb *strings.Builder) error {
-	sb.WriteString(" ORDER BY ts DESC")
-	return nil
-}
-
 // BusEventFilter is a filter for bus events.
 type BusEventFilter struct {
-	Names          []string
-	TimerangeStart time.Time
-	TimerangeEnd   time.Time
+	Names     []string  // The name(s) of packets to filter for
+	StartTime time.Time // Starting time range. All packets >= StartTime
+	EndTime   time.Time // Ending time range. All packets <= EndTime
+	Indexes   []int     // The specific index of the packets to index.
 }
 
 // now we can optionally add a limit.
 
-func (tdb *TelemDb) GetPackets(ctx context.Context, filter BusEventFilter, options ...QueryModifier) ([]skylab.BusEvent, error) {
+func (tdb *TelemDb) GetPackets(ctx context.Context, filter BusEventFilter, lim *LimitOffsetModifier) ([]skylab.BusEvent, error) {
 	// construct a simple
 	var whereFrags = make([]string, 0)
 
 	// if we're filtering by names, add a where clause for it.
 	if len(filter.Names) > 0 {
-		names := strings.Join(filter.Names, ", ")
+		// we have to quote our individual names
+		names := strings.Join(filter.Names, `", "`)
 		qString := fmt.Sprintf(`name IN ("%s")`, names)
 		whereFrags = append(whereFrags, qString)
 	}
@@ -178,18 +167,27 @@ func (tdb *TelemDb) GetPackets(ctx context.Context, filter BusEventFilter, optio
 	// using BETWEEN since apparenlty that can be better?
 
 	// next, check if we have a start/end time, add constraints
-	if !filter.TimerangeEnd.IsZero() {
-		qString := fmt.Sprintf("ts <= %d", filter.TimerangeEnd.UnixMilli())
+	if !filter.EndTime.IsZero() {
+		qString := fmt.Sprintf("ts <= %d", filter.EndTime.UnixMilli())
 		whereFrags = append(whereFrags, qString)
 	}
-	if !filter.TimerangeStart.IsZero() {
+	if !filter.StartTime.IsZero() {
 		// we have an end range
-		qString := fmt.Sprintf("ts >= %d", filter.TimerangeStart.UnixMilli())
+		qString := fmt.Sprintf("ts >= %d", filter.StartTime.UnixMilli())
+		whereFrags = append(whereFrags, qString)
+	}
+	if len(filter.Indexes) > 0 {
+		s := make([]string, 0)
+		for _, idx := range filter.Indexes {
+			s = append(s, fmt.Sprint(idx))
+		}
+		idxs := strings.Join(s, ", ")
+		qString := fmt.Sprintf(`idx in (%s)`, idxs)
 		whereFrags = append(whereFrags, qString)
 	}
 
 	sb := strings.Builder{}
-	sb.WriteString(`SELECT * from "bus_events"`)
+	sb.WriteString(`SELECT ts, name, data from "bus_events"`)
 	// construct the full statement.
 	if len(whereFrags) > 0 {
 		// use the where clauses.
@@ -197,10 +195,12 @@ func (tdb *TelemDb) GetPackets(ctx context.Context, filter BusEventFilter, optio
 		sb.WriteString(strings.Join(whereFrags, " AND "))
 	}
 
+	sb.WriteString(" ORDER BY ts DESC")
+
 	// Augment our data further if there's i.e a limit modifier.
 	// TODO: factor this out maybe?
-	for _, m := range options {
-		m.ModifyStatement(&sb)
+	if lim != nil {
+		lim.ModifyStatement(&sb)
 	}
 	rows, err := tdb.db.QueryxContext(ctx, sb.String())
 	if err != nil {
@@ -246,37 +246,45 @@ type Datum struct {
 // GetValues queries the database for values in a given time range.
 // A value is a specific data point. For example, bms_measurement.current
 // would be a value.
-func (tdb *TelemDb) GetValues(ctx context.Context, bef BusEventFilter,
-	field string, opts ...QueryModifier) ([]Datum, error) {
+func (tdb *TelemDb) GetValues(ctx context.Context, filter BusEventFilter,
+	field string, lim *LimitOffsetModifier) ([]Datum, error) {
 	// this fragment uses json_extract from sqlite to get a single
 	// nested value.
 	sb := strings.Builder{}
 	sb.WriteString(`SELECT ts as timestamp, json_extract(data, '$.' || ?) as val FROM bus_events WHERE `)
-	if len(bef.Names) != 1 {
+	if len(filter.Names) != 1 {
 		return nil, errors.New("invalid number of names")
 	}
+	whereFrags := []string{"name is ?"}
 
-	qStrings := []string{"name is ?"}
-	// add timestamp limit.
-	if !bef.TimerangeStart.IsZero() {
-		qString := fmt.Sprintf("ts >= %d", bef.TimerangeStart.UnixMilli())
-		qStrings = append(qStrings, qString)
+	if !filter.StartTime.IsZero() {
+		qString := fmt.Sprintf("ts >= %d", filter.StartTime.UnixMilli())
+		whereFrags = append(whereFrags, qString)
 	}
 
-	if !bef.TimerangeEnd.IsZero() {
-		qString := fmt.Sprintf("ts <= %d", bef.TimerangeEnd.UnixMilli())
-		qStrings = append(qStrings, qString)
+	if !filter.EndTime.IsZero() {
+		qString := fmt.Sprintf("ts <= %d", filter.EndTime.UnixMilli())
+		whereFrags = append(whereFrags, qString)
+	}
+	if len(filter.Indexes) > 0 {
+		s := make([]string, 0)
+		for _, idx := range filter.Indexes {
+			s = append(s, fmt.Sprint(idx))
+		}
+		idxs := strings.Join(s, ", ")
+		qString := fmt.Sprintf(`idx in (%s)`, idxs)
+		whereFrags = append(whereFrags, qString)
 	}
 	// join qstrings with AND
-	sb.WriteString(strings.Join(qStrings, " AND "))
+	sb.WriteString(strings.Join(whereFrags, " AND "))
 
-	for _, m := range opts {
-		if m == nil {
-			continue
-		}
-		m.ModifyStatement(&sb)
+	sb.WriteString(" ORDER BY ts DESC")
+
+	if lim != nil {
+		lim.ModifyStatement(&sb)
 	}
-	rows, err := tdb.db.QueryxContext(ctx, sb.String(), field, bef.Names[0])
+
+	rows, err := tdb.db.QueryxContext(ctx, sb.String(), field, filter.Names[0])
 	if err != nil {
 		return nil, err
 	}
@@ -299,40 +307,92 @@ func (tdb *TelemDb) GetValues(ctx context.Context, bef BusEventFilter,
 	return data, nil
 }
 
-// PacketDef is a database packet model
-type PacketDef struct {
-	Name        string
-	Description string
-	Id          int
+// AddDocument inserts a new document to the store if it is unique and valid.
+func (tdb *TelemDb) AddDocument(ctx context.Context, obj json.RawMessage) error {
+	const insertStmt = `INSERT INTO openmct_objects (data) VALUES (json(?))`
+	_, err := tdb.db.ExecContext(ctx, insertStmt, obj)
+	return err
 }
 
-type FieldDef struct {
-	Name    string
-	SubName string
-	Packet  string
-	Type    string
+// DocumentNotFoundError is when the underlying document cannot be found.
+type DocumentNotFoundError string
+
+func (e DocumentNotFoundError) Error() string {
+	return fmt.Sprintf("document could not find key: %s", string(e))
 }
 
-// PacketNotFoundError is when a matching packet cannot be found.
-type PacketNotFoundError string
 
-func (e *PacketNotFoundError) Error() string {
-	return "packet not found: " + string(*e)
+// UpdateDocument replaces the entire contents of a document matching
+// the given key. Note that the key is derived from the document,
+// and no checks are done to ensure that the new key is the same.
+func (tdb *TelemDb) UpdateDocument(ctx context.Context, key string,
+	obj json.RawMessage) error {
+
+	const upd = `UPDATE openmct_objects SET data = json(?) WHERE key IS ?`
+	r, err := tdb.db.ExecContext(ctx, upd, obj, key)
+	if err != nil {
+		return err
+	}
+	n, err := r.RowsAffected()
+	if err != nil {
+		return err
+	}
+	if n != 1 {
+		return DocumentNotFoundError(key)
+	}
+	return err
 }
 
-// GetPacketDefN retrieves a packet matching the given name, if it exists.
-// returns PacketNotFoundError if a matching packet could not be found.
-func (tdb *TelemDb) GetPacketDefN(name string) (*PacketDef, error) {
-	return nil, nil
+
+// GetDocument gets the document matching the corresponding key.
+func (tdb *TelemDb) GetDocument(ctx context.Context, key string) (json.RawMessage, error) {
+	const get = `SELECT data FROM openmct_objects WHERE key IS ?`
+
+	row := tdb.db.QueryRowxContext(ctx, get, key)
+
+	var res []byte // VERY important, json.RawMessage won't work here
+	// since the scan function does not look at underlying types.
+	row.Scan(&res)
+
+	if len(res) == 0 {
+		return nil, DocumentNotFoundError(key)
+	}
+
+	return res, nil
 }
 
-// GetPacketDefF retrieves the parent packet for a given field.
-// This function cannot return PacketNotFoundError since we have SQL FKs enforcing.
-func (tdb *TelemDb) GetPacketDefF(field FieldDef) (*PacketDef, error) {
-	return nil, nil
+// GetAllDocuments returns all documents in the database.
+func (tdb *TelemDb) GetAllDocuments(ctx context.Context) ([]json.RawMessage, error) {
+	const getall = `SELECT data FROM openmct_objects`;
+
+	rows, err := tdb.db.QueryxContext(ctx, getall)
+	defer rows.Close()
+	if err != nil {
+		return nil, err
+	}
+	docs := make([]json.RawMessage, 0)
+	for rows.Next() {
+		var j json.RawMessage
+		rows.Scan(&j)
+		docs = append(docs, j)
+	}
+	return docs, nil
 }
 
-// GetFieldDefs returns the given fields for a given packet definition.
-func (tdb *TelemDb) GetFieldDefs(pkt PacketDef) ([]FieldDef, error) {
-	return nil, nil
+// DeleteDocument removes a document from the store, or errors
+// if it does not exist.
+func (tdb *TelemDb) DeleteDocument(ctx context.Context, key string) error {
+	const del = `DELETE FROM openmct_objects WHERE key IS ?`
+	res, err := tdb.db.ExecContext(ctx, del, key)
+	if err != nil {
+		return err
+	}
+	n, err := res.RowsAffected()
+	if err != nil {
+		return err
+	}
+	if n != 1 {
+		return DocumentNotFoundError(key)
+	}
+	return err
 }

db_test.go

@@ -3,7 +3,11 @@ package gotelem
 import (
 	"bufio"
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"math/rand"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -85,7 +89,6 @@ func MakeMockDatabase(name string) *TelemDb {
 
 func TestTelemDb(t *testing.T) {
 
-
 	t.Run("test opening database", func(t *testing.T) {
 		// create our mock
 		tdb := MakeMockDatabase(t.Name())
@@ -154,3 +157,128 @@ func TestTelemDb(t *testing.T) {
 
 	})
 }
+
+func MockDocument(key string) json.RawMessage {
+	var v = make(map[string]interface{})
+
+	v["identifier"] = map[string]string{"key": key}
+	v["randomdata"] = rand.Int()
+	res, err := json.Marshal(v)
+	if err != nil {
+		panic(err)
+	}
+	return res
+}
+
+func TestDbDocuments(t *testing.T) {
+
+	t.Run("test inserting a document", func(t *testing.T) {
+		tdb := MakeMockDatabase(t.Name())
+		tdb.db.Ping()
+		ctx := context.Background()
+		err := tdb.AddDocument(ctx, MockDocument("hi"))
+		if err != nil {
+			t.Fatalf("AddDocument expected no error, got err=%v", err)
+		}
+	})
+
+	t.Run("test inserting duplicate documents", func(t *testing.T) {
+		tdb := MakeMockDatabase(t.Name())
+		tdb.db.Ping()
+		ctx := context.Background()
+		doc := MockDocument("hi")
+		err := tdb.AddDocument(ctx, doc)
+		if err != nil {
+			t.Fatalf("AddDocument expected no error, got err=%v", err)
+		}
+
+		err = tdb.AddDocument(ctx, doc)
+		if err == nil {
+			t.Fatalf("AddDocument expected duplicate key error, got nil")
+		}
+	})
+	t.Run("test inserting bad document", func(t *testing.T) {
+		tdb := MakeMockDatabase(t.Name())
+		tdb.db.Ping()
+		ctx := context.Background()
+		var badDoc = map[string]string{"bad":"duh"}
+		msg, err := json.Marshal(badDoc)
+		if err != nil {
+			panic(err)
+		}
+		err = tdb.AddDocument(ctx, msg)
+
+		if err == nil {
+			t.Fatalf("AddDocument expected error, got nil")
+		}
+
+	})
+
+	t.Run("test getting document", func(t *testing.T) {
+		tdb := MakeMockDatabase(t.Name())
+		tdb.db.Ping()
+		ctx := context.Background()
+		doc := MockDocument("hi")
+		err := tdb.AddDocument(ctx, doc)
+		if err != nil {
+			t.Fatalf("AddDocument expected no error, got err=%v", err)
+		}
+
+		res, err := tdb.GetDocument(ctx, "hi")
+		if err != nil {
+			t.Fatalf("GetDocument expected no error, got err=%v", err)
+		}
+		if !reflect.DeepEqual(res, doc) {
+			t.Fatalf("GetDocument did not return identical document")
+		}
+
+	})
+
+	t.Run("test getting nonexistent document", func(t *testing.T) {
+		tdb := MakeMockDatabase(t.Name())
+		tdb.db.Ping()
+		ctx := context.Background()
+
+		res, err := tdb.GetDocument(ctx, "hi")
+
+		if err == nil || !errors.Is(err, DocumentNotFoundError("hi")){
+			t.Fatalf("GetDocument expected DocumentNotFoundError, got %v", err)
+		}
+		if res != nil {
+			t.Fatalf("GetDocument expected nil result, got %v", res)
+		}
+	})
+
+	t.Run("test update document", func(t *testing.T) {
+		tdb := MakeMockDatabase(t.Name())
+		tdb.db.Ping()
+		ctx := context.Background()
+		doc1 := MockDocument("hi")
+		doc2 := MockDocument("hi") // same key, we want to update.
+
+		tdb.AddDocument(ctx, doc1)
+		err := tdb.UpdateDocument(ctx, "hi", doc2)
+		if err != nil {
+			t.Fatalf("UpdateDocument expected no error, got err=%v", err)
+		}
+
+		// compare.
+		res, _ := tdb.GetDocument(ctx, "hi")
+		if !reflect.DeepEqual(res, doc2) {
+			t.Fatalf("UpdateDocument did not return new doc, got %s", res)
+		}
+
+	})
+
+	t.Run("test update nonexistent document", func(t *testing.T) {
+		tdb := MakeMockDatabase(t.Name())
+		tdb.db.Ping()
+		ctx := context.Background()
+		doc := MockDocument("hi")
+		err := tdb.UpdateDocument(ctx, "badKey", doc)
+		if err == nil {
+			t.Fatalf("UpdateDocument expected error, got nil")
+		}
+	})
+
+}

http.go

@@ -31,7 +31,7 @@ func extractBusEventFilter(r *http.Request) (*BusEventFilter, error) {
 		if err != nil {
 			return bef, err
 		}
-		bef.TimerangeStart = t
+		bef.StartTime = t
 	}
 	if el := v.Get("end"); el != "" {
 		// parse the start time query.
@@ -39,7 +39,15 @@ func extractBusEventFilter(r *http.Request) (*BusEventFilter, error) {
 		if err != nil {
 			return bef, err
 		}
-		bef.TimerangeEnd = t
+		bef.EndTime = t
+	}
+	bef.Indexes = make([]int, 0)
+	for _, strIdx := range v["idx"] {
+		idx, err := strconv.ParseInt(strIdx, 10, 32)
+		if err != nil {
+			return nil, err
+		}
+		bef.Indexes = append(bef.Indexes, int(idx))
 	}
 	return bef, nil
 }
@@ -132,16 +140,21 @@ func apiV1(broker *Broker, tdb *TelemDb) chi.Router {
 
 	})
 
-	// records are driving segments/runs.
-	r.Route("/records", func(r chi.Router) {
-		r.Get("/", apiV1GetRecords(tdb))            // get all runs
-		r.Get("/active", apiV1GetActiveRecord(tdb)) // get current run (no end time)
-		r.Post("/", apiV1StartRecord(tdb))          // create a new run (with note). Ends active run if any, and creates new active run (no end time)
-		r.Get("/{id}", apiV1GetRecord(tdb))         // get details on a specific run
-		r.Put("/{id}", apiV1UpdateRecord(tdb))      // update a specific run. Can only be used to add notes/metadata, and not to change time/id.
+	// OpenMCT domain object storage. Basically an arbitrary JSON document store
 
+	r.Route("/openmct", func(r chi.Router) {
+		// key is a column on our json store, it's nested under identifier.key
+		r.Get("/{key}", func(w http.ResponseWriter, r *http.Request) {})
+		r.Put("/{key}", func(w http.ResponseWriter, r *http.Request) {})
+		r.Delete("/{key}", func(w http.ResponseWriter, r *http.Request) {})
+		// create a new object.
+		r.Post("/", func(w http.ResponseWriter, r *http.Request) {})
+		// subscribe to object updates.
+		r.Get("/subscribe", func(w http.ResponseWriter, r *http.Request) {})
 	})
 
+	// records are driving segments/runs.
+
 	r.Get("/stats", func(w http.ResponseWriter, r *http.Request) {
 
 	}) // v1 api stats (calls, clients, xbee connected, meta health ok)
@@ -215,16 +228,13 @@ func apiV1GetPackets(tdb *TelemDb) http.HandlerFunc {
 		lim, err := extractLimitModifier(r)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
+			fmt.Print(lim)
 			return
 		}
 
 		// TODO: is the following check needed?
 		var res []skylab.BusEvent
-		if lim != nil {
-			res, err = tdb.GetPackets(r.Context(), *bef, lim)
-		} else {
-			res, err = tdb.GetPackets(r.Context(), *bef)
-		}
+		res, err = tdb.GetPackets(r.Context(), *bef, lim)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
@@ -264,15 +274,10 @@ func apiV1GetValues(db *TelemDb) http.HandlerFunc {
 		// override the bus event filter name option
 		bef.Names = []string{name}
 
-		var order = &OrderByTimestampModifer{}
 
 		var res []Datum
 		// make the call, skip the limit modifier if it's nil.
-		if lim == nil {
-			res, err = db.GetValues(r.Context(), *bef, field, order)
-		} else {
-			res, err = db.GetValues(r.Context(), *bef, field, lim, order)
-		}
+		res, err = db.GetValues(r.Context(), *bef, field, lim)
 		if err != nil {
 			// 500 server error:
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -287,28 +292,3 @@ func apiV1GetValues(db *TelemDb) http.HandlerFunc {
 	}
 
 }
-
-// TODO: rename. record is not a clear name. Runs? drives? segments?
-func apiV1GetRecords(db *TelemDb) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-
-	}
-}
-
-func apiV1GetActiveRecord(db *TelemDb) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-
-	}
-}
-
-func apiV1StartRecord(db *TelemDb) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}
-
-func apiV1GetRecord(db *TelemDb) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}
-
-func apiV1UpdateRecord(db *TelemDb) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {}
-}

migrations/2_addl_tables_down.sql

@@ -1,2 +1 @@
 DROP TABLE "drive_records";
-DROP TABLE "position_logs";

migrations/2_addl_tables_up.sql

@@ -7,12 +7,3 @@ CREATE TABLE "drive_records" (
 	PRIMARY KEY("id" AUTOINCREMENT),
 	CONSTRAINT "duration_valid" CHECK(end_time is null or start_time < end_time)
 );
-
-CREATE TABLE "position_logs" (
-	"ts"	INTEGER NOT NULL,
-	"source"	TEXT NOT NULL,
-	"lat"	REAL NOT NULL,
-	"lon"	REAL NOT NULL,
-	"elevation"	REAL,
-	CONSTRAINT "no_empty_source" CHECK(source is not "")
-);

migrations/6_bus_index_column_down.sql

@@ -0,0 +1 @@
+ALTER TABLE "bus_events" DROP COLUMN idx;

migrations/6_bus_index_column_up.sql

@@ -0,0 +1 @@
+ALTER TABLE "bus_events" ADD COLUMN idx GENERATED ALWAYS AS (json_extract(data, '$.idx')) VIRTUAL;

migrations/7_domainobject_down.sql

@@ -0,0 +1,2 @@
+DROP TABLE openmct_objects;
+DROP INDEX openmct_key;

migrations/7_domainobject_up.sql

@@ -0,0 +1,6 @@
+CREATE TABLE openmct_objects (
+	data TEXT,
+	key TEXT GENERATED ALWAYS AS (json_extract(data, '$.identifier.key')) VIRTUAL UNIQUE NOT NULL
+);
+-- fast key-lookup
+CREATE INDEX openmct_key on openmct_objects(key);