refactor(extmarks): use a more efficient representation

marktree.c was originally constructed as a "generic" datatype,
to make the prototyping of its internal logic as simple as possible
and also as the usecases for various kinds of extmarks/decorations was not yet decided.
As a consequence of this, various extra indirections and allocations was
needed to use marktree to implement extmarks (ns/id pairs) and
decorations of different kinds (some which is just a single highlight
id, other an allocated list of virtual text/lines)

This change removes a lot of indirection, by making Marktree specialized
for the usecase. In particular, the namespace id and mark id is stored
directly, instead of the 64-bit global id particular to the Marktree
struct. This removes the two maps needed to convert between global and
per-ns ids.

Also, "small" decorations are stored inline, i.e. those who
doesn't refer to external heap memory anyway. That is highlights (with
priority+flags) are stored inline, while virtual text, which anyway
occurs a lot of heap allocations, do not. (previously a hack was used
to elide heap allocations for highlights with standard prio+flags)

TODO(bfredl): the functionaltest-lua CI version of gcc is having
severe issues with uint16_t bitfields, so splitting up compound
assignments and redundant casts are needed. Clean this up once we switch
to a working compiler version.

src/nvim/marktree.h

@@ -3,8 +3,10 @@
 
 #include <stdint.h>
 
+#include "nvim/assert.h"
 #include "nvim/garray.h"
 #include "nvim/map.h"
+#include "nvim/types.h"
 #include "nvim/pos.h"
 
 #define MT_MAX_DEPTH 20
@@ -15,13 +17,6 @@ typedef struct {
   int32_t col;
 } mtpos_t;
 
-typedef struct {
-  int32_t row;
-  int32_t col;
-  uint64_t id;
-  bool right_gravity;
-} mtmark_t;
-
 typedef struct mtnode_s mtnode_t;
 typedef struct {
   int oldcol;
@@ -39,12 +34,75 @@ typedef struct {
 
 // Internal storage
 //
-// NB: actual marks have id > 0, so we can use (row,col,0) pseudo-key for
+// NB: actual marks have flags > 0, so we can use (row,col,0) pseudo-key for
 // "space before (row,col)"
 typedef struct {
   mtpos_t pos;
-  uint64_t id;
+  uint32_t ns;
+  uint32_t id;
+  int32_t hl_id;
+  uint16_t flags;
+  uint16_t priority;
+  Decoration *decor_full;
 } mtkey_t;
+#define MT_INVALID_KEY (mtkey_t) { { -1, -1 }, 0, 0, 0, 0, 0, NULL }
+
+#define MT_FLAG_REAL (((uint16_t)1) << 0)
+#define MT_FLAG_END (((uint16_t)1) << 1)
+#define MT_FLAG_PAIRED (((uint16_t)1) << 2)
+#define MT_FLAG_HL_EOL (((uint16_t)1) << 3)
+
+#define DECOR_LEVELS 4
+#define MT_FLAG_DECOR_OFFSET 4
+#define MT_FLAG_DECOR_MASK (((uint16_t)(DECOR_LEVELS-1)) << MT_FLAG_DECOR_OFFSET)
+
+// next flag is (((uint16_t)1) << 6)
+
+// These _must_ be last to preserve ordering of marks
+#define MT_FLAG_RIGHT_GRAVITY (((uint16_t)1) << 14)
+#define MT_FLAG_LAST (((uint16_t)1) << 15)
+
+#define MT_FLAG_EXTERNAL_MASK (MT_FLAG_DECOR_MASK | MT_FLAG_RIGHT_GRAVITY | MT_FLAG_HL_EOL)
+
+#define MARKTREE_END_FLAG (((uint64_t)1) << 63)
+static inline uint64_t mt_lookup_id(uint32_t ns, uint32_t id, bool enda)
+{
+  return (uint64_t)ns << 32 | id | (enda?MARKTREE_END_FLAG:0);
+}
+#undef MARKTREE_END_FLAG
+
+static inline uint64_t mt_lookup_key(mtkey_t key)
+{
+  return mt_lookup_id(key.ns, key.id, key.flags & MT_FLAG_END);
+}
+
+static inline bool mt_paired(mtkey_t key)
+{
+  return key.flags & MT_FLAG_PAIRED;
+}
+
+static inline bool mt_end(mtkey_t key)
+{
+  return key.flags & MT_FLAG_END;
+}
+
+static inline bool mt_right(mtkey_t key)
+{
+  return key.flags & MT_FLAG_RIGHT_GRAVITY;
+}
+
+static inline uint8_t marktree_decor_level(mtkey_t key)
+{
+  return (uint8_t)((key.flags&MT_FLAG_DECOR_MASK) >> MT_FLAG_DECOR_OFFSET);
+}
+
+static inline uint16_t mt_flags(bool right_gravity, uint8_t decor_level)
+{
+  assert(decor_level < DECOR_LEVELS);
+  return (uint16_t)((right_gravity ? MT_FLAG_RIGHT_GRAVITY : 0)
+                    | (decor_level << MT_FLAG_DECOR_OFFSET));
+}
+
 
 struct mtnode_s {
   int32_t n;
@@ -61,7 +119,6 @@ struct mtnode_s {
 typedef struct {
   mtnode_t *root;
   size_t n_keys, n_nodes;
-  uint64_t next_id;
   // TODO(bfredl): the pointer to node could be part of the larger
   // Map(uint64_t, ExtmarkItem) essentially;
   PMap(uint64_t) id2node[1];
@@ -72,16 +129,4 @@ typedef struct {
 # include "marktree.h.generated.h"
 #endif
 
-#define MARKTREE_PAIRED_FLAG (((uint64_t)1) << 1)
-#define MARKTREE_END_FLAG (((uint64_t)1) << 0)
-
-#define DECOR_LEVELS 4
-#define DECOR_OFFSET 61
-#define DECOR_MASK (((uint64_t)(DECOR_LEVELS-1)) << DECOR_OFFSET)
-
-static inline uint8_t marktree_decor_level(uint64_t id)
-{
-  return (uint8_t)((id&DECOR_MASK) >> DECOR_OFFSET);
-}
-
 #endif  // NVIM_MARKTREE_H