Brought in code by Gemini for handling lists and Task lists

lists and task lists are a PITA to handle, I may need to handle them
myself in the future but that'll probably become clear via testing
rather than simply doing it as is because the parsing logic is annoying

psb.cabal

@@ -27,7 +27,7 @@ common warnings
 
 library
     hs-source-dirs: src
-    exposed-modules: Djot Markdown HTML Logger IR Logger.Shake Psb.Main Utilities Utilities.FilePath Utilities.Action Utilities.Javascript Utilities.CSS Templates Types Config Utilities.Bundling
+    exposed-modules: Djot Djot.Lists Markdown HTML Logger IR Logger.Shake Psb.Main Utilities Utilities.FilePath Utilities.Action Utilities.Javascript Utilities.CSS Templates Types Config Utilities.Bundling
     other-modules: Utilities.Parsing
     build-depends:    base, mustache >=2.4.2, shake >= 0.19.8, deriving-aeson >= 0.2.9, aeson, text >= 2.1.2, time, unordered-containers, yaml, megaparsec >= 9.7.0, transformers >= 0.6.2, bytestring
     default-extensions: ApplicativeDo DataKinds NamedFieldPuns DerivingVia LambdaCase TypeApplications DeriveGeneric OverloadedRecordDot NamedFieldPuns DuplicateRecordFields DisambiguateRecordFields FlexibleInstances

src/Djot.hs

@@ -19,6 +19,7 @@ import qualified Data.List.NonEmpty as NE
 import Data.STRef (modifySTRef', newSTRef, readSTRef, writeSTRef)
 import Data.Text (Text)
 import qualified Data.Text as T
+import qualified Djot.Lists
 import IR
 import Logger (Logger (logCallStack, logDebug, logError))
 import Text.Megaparsec (MonadParsec (lookAhead, notFollowedBy, parseError, try), ParseErrorBundle (ParseErrorBundle, bundleErrors), SourcePos (sourceColumn), anySingle, choice, errorOffset, getInput, getOffset, manyTill, parse, satisfy, sepBy, setErrorOffset, someTill)
@@ -40,9 +41,8 @@ blockElement accumulated_attributes =
   choice
     [ lookAhead (char '#') *> header accumulated_attributes,
       lookAhead (char '>') *> blockQuote accumulated_attributes,
-      lookAhead listMarker
+      try $ taskListBlock accumulated_attributes,
-        *> (lookAhead (bullet *> taskMarker) *> taskListBlock accumulated_attributes)
+      try $ listBlock accumulated_attributes,
-        <|> listBlock accumulated_attributes,
       lookAhead codeFence
         *> (lookAhead (codeFence *> rawLang) *> rawBlock accumulated_attributes)
         <|> codeBlock accumulated_attributes,
@@ -100,21 +100,8 @@ blockQuote attrs = do
 
 listBlock :: (Logger m, Characters s) => Attrs -> Parser s m Element
 listBlock attrs = do
-  error "this is probably fundamentally wrong and needs to be redone to work"
+  list <- Djot.Lists.djotList
-  startOffset <- getOffset
+  error ""
-  list_type <- listMarker
-  first_item <- list_item_content
-  rem_items <- manyTill (listMarker' list_type *> list_item_content) $ lookAhead blockSeparator
-  let items = first_item : rem_items
-  case parse listBlock' "" $ error "todo: figure out how to take our list of items and put them toegether for easy parsing" of
-    Right ret -> pure ret
-    Left (ParseErrorBundle errs _) -> do
-      logError "Error in blockQuote offset may be off"
-      let remap err = setErrorOffset (errorOffset err + startOffset) err
-      parseError $ remap $ NE.head errs
-  where
-    list_item_content = error "todo"
-    listBlock' = error "todo"
 
 taskListBlock :: (Logger m, Characters s) => Attrs -> Parser s m Element
 taskListBlock attrs = do

src/Djot/Lists.hs

@@ -0,0 +1,221 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+-- This is written by Gemini barring minor adjustments made manually to fit
+-- into the Parsing utilities and logger stuff
+
+module Djot.Lists (List, djotList, TaskList, djotTaskList) where
+
+import Data.Functor (void)
+import Data.Text (Text)
+import qualified Data.Text as T
+import Data.Void
+import Logger (Logger (logCallStack, logDebug, logError))
+import Text.Megaparsec
+import Text.Megaparsec.Char
+import qualified Text.Megaparsec.Char.Lexer as L
+import Utilities.Parsing
+
+-- | Djot supports three types of ordered list delimiters.
+data Delimiter = Period | RightParen | Enclosed -- e.g., '1.', '1)', '(1)'
+  deriving (Show, Eq)
+
+-- | The style of the list marker.
+data MarkerStyle
+  = Bullet Char -- '*', '-', '+'
+  | Decimal Delimiter -- '1'
+  | LowerAlpha Delimiter -- 'a'
+  | UpperAlpha Delimiter -- 'A'
+  | LowerRoman Delimiter -- 'i'
+  | UpperRoman Delimiter -- 'I'
+  deriving (Show, Eq)
+
+-- | A single item in a list.
+data ListItem = ListItem
+  { itemStyle :: MarkerStyle,
+    itemIndent :: Int, -- Column where the block content starts
+    itemContent :: [Text] -- Simplified: usually this would be [Block]
+  }
+  deriving (Show, Eq)
+
+-- | A full list is a collection of items sharing the exact same MarkerStyle.
+data List = List MarkerStyle [ListItem]
+  deriving (Show, Eq)
+
+-- | Parses whitespace but does not consume newlines.
+sc :: (Logger m, Characters s) => Parser s m ()
+sc = hidden (void $ takeWhileP (Just "space") (== ' '))
+
+-- | Parses the punctuation after/around an ordered list number/letter.
+-- E.g., takes a parser for the numeral (like `lowerRoman`) and applies delimiters.
+orderedDelimiter :: (Logger m, Characters s) => Parser s m a -> Parser s m (a, Delimiter)
+orderedDelimiter inner =
+  (try (between (char '(') (char ')') inner) >>= \x -> return (x, Enclosed))
+    <|> (try (inner <* char ')') >>= \x -> return (x, RightParen))
+    <|> (try (inner <* char '.') >>= \x -> return (x, Period))
+
+-- | Parses a bullet marker.
+bulletMarker :: (Logger m, Characters s) => Parser s m MarkerStyle
+bulletMarker = Bullet <$> oneOf ['*', '-', '+']
+
+-- | Decimals: '1.', '1)', '(1)'
+decimalMarker :: (Logger m, Characters s) => Parser s m MarkerStyle
+decimalMarker = Decimal . snd <$> orderedDelimiter L.decimal
+
+-- | Lower/Upper Alpha: 'a.', 'A)', etc.
+-- (Note: In a full implementation, you must ensure these are single letters)
+lowerAlphaMarker :: (Logger m, Characters s) => Parser s m MarkerStyle
+lowerAlphaMarker = LowerAlpha . snd <$> orderedDelimiter lowerChar
+
+upperAlphaMarker :: (Logger m, Characters s) => Parser s m MarkerStyle
+upperAlphaMarker = UpperAlpha . snd <$> orderedDelimiter upperChar
+
+-- | Lower/Upper Roman
+-- Uses a simplified roman numeral regex/parser for illustration.
+romanLower :: (Logger m, Characters s) => Parser s m Text
+romanLower = toText <$> takeWhile1P (Just "lower roman") (`elem` ['i', 'v', 'x', 'l', 'c', 'd', 'm'])
+
+romanUpper :: (Logger m, Characters s) => Parser s m Text
+romanUpper = toText <$> takeWhile1P (Just "upper roman") (`elem` ['I', 'V', 'X', 'L', 'C', 'D', 'M'])
+
+lowerRomanMarker :: (Logger m, Characters s) => Parser s m MarkerStyle
+lowerRomanMarker = LowerRoman . snd <$> orderedDelimiter romanLower
+
+upperRomanMarker :: (Logger m, Characters s) => Parser s m MarkerStyle
+upperRomanMarker = UpperRoman . snd <$> orderedDelimiter romanUpper
+
+-- | Attempts to parse any valid Djot list marker.
+listMarker :: (Logger m, Characters s) => Parser s m MarkerStyle
+listMarker =
+  choice
+    [ bulletMarker,
+      decimalMarker,
+      lowerRomanMarker, -- Order matters here to prevent 'i.' parsing as LowerAlpha
+      lowerAlphaMarker,
+      upperRomanMarker,
+      upperAlphaMarker
+    ]
+
+-- | Parses a single list item.
+listItem :: (Logger m, Characters s) => Parser s m ListItem
+listItem = do
+  -- 1. Grab the current indentation level before the item
+  startPos <- L.indentLevel
+
+  -- 2. Parse the marker
+  marker <- listMarker
+
+  -- 3. Consume spaces between marker and content (mandatory in Djot)
+  void $ some (char ' ')
+
+  -- 4. Find the column where the actual content begins.
+  -- Djot relies on this column to group subsequent lines.
+  contentPos <- L.indentLevel
+
+  -- 5. Parse the block content based on indentation.
+  -- We use `L.indentBlock` or manually grab lines that are indented >= contentPos.
+  content <- some (indentedLine contentPos)
+
+  return $ ListItem marker (unPos contentPos) content
+
+-- | Helper to grab a line that respects the item's indentation level.
+indentedLine :: (Logger m, Characters s) => Pos -> Parser s m Text
+indentedLine minIndent = do
+  currentIndent <- L.indentLevel
+  if currentIndent >= minIndent
+    then do
+      line <- takeWhileP (Just "character") (/= '\n')
+      void (char '\n') <|> eof
+      return $ toText line
+    else fail "Line not indented enough for list item"
+
+-- | Parses a complete list by ensuring all subsequent items share the same MarkerStyle.
+djotList :: (Logger m, Characters s) => Parser s m List
+djotList = do
+  firstItem <- listItem
+  let style = itemStyle firstItem
+
+  -- Try to parse more items, but ONLY if they have the exact same style.
+  restItems <- many (try $ itemWithStyle style)
+
+  return $ List style (firstItem : restItems)
+
+-- | Parses an item only if it matches the expected style.
+itemWithStyle :: (Logger m, Characters s) => MarkerStyle -> Parser s m ListItem
+itemWithStyle expectedStyle = do
+  item <- listItem
+  if itemStyle item == expectedStyle
+    then return item
+    else fail "List marker style mismatch"
+
+-- | Represents the state of the task checkbox.
+data TaskStatus = Unchecked | Checked
+  deriving (Show, Eq)
+
+-- | A specific list item that is guaranteed to be a task.
+data TaskListItem = TaskListItem
+  { taskStyle :: MarkerStyle,
+    taskIndent :: Int,
+    taskStatus :: TaskStatus,
+    taskContent :: [Text]
+  }
+  deriving (Show, Eq)
+
+-- | A list composed entirely of TaskListItems.
+data TaskList = TaskList MarkerStyle [TaskListItem]
+  deriving (Show, Eq)
+
+-- | Parses the Djot task checkbox and returns its status.
+taskCheckbox :: (Logger m, Characters s) => Parser s m TaskStatus
+taskCheckbox = do
+  void $ char '['
+  status <-
+    (char ' ' *> pure Unchecked)
+      <|> (oneOf ['x', 'X'] *> pure Checked)
+  void $ char ']'
+
+  -- Djot requires a space after the checkbox (or a newline if the item is blank)
+  void (char ' ') <|> lookAhead (void $ char '\n') <|> eof
+
+  pure status
+
+-- | Parses a single task list item.
+taskListItem :: (Logger m, Characters s) => Parser s m TaskListItem
+taskListItem = do
+  _startPos <- L.indentLevel
+
+  -- 1. Parse the list marker (bullets or ordered)
+  marker <- listMarker
+
+  -- 2. Consume mandatory spaces after the marker
+  void $ some (char ' ')
+
+  -- 3. Capture the indentation level right HERE.
+  -- The '[' of the checkbox dictates the column that subsequent lines must respect.
+  contentPos <- L.indentLevel
+
+  -- 4. Parse the mandatory checkbox (if this fails, it's not a task list item)
+  status <- taskCheckbox
+
+  -- 5. Parse the content, ensuring continuation lines respect the `contentPos`
+  -- (Using the same `indentedLine` helper from the previous parser)
+  content <- some (indentedLine contentPos)
+
+  pure $ TaskListItem marker (unPos contentPos) status content
+
+-- | Parses a complete task list, ensuring consistent marker styles.
+djotTaskList :: (Logger m, Characters s) => Parser s m TaskList
+djotTaskList = do
+  firstItem <- taskListItem
+  let style = taskStyle firstItem
+
+  restItems <- many (try $ taskItemWithStyle style)
+
+  pure $ TaskList style (firstItem : restItems)
+
+-- | Helper to strictly enforce uniform marker styles across the task list.
+taskItemWithStyle :: (Logger m, Characters s) => MarkerStyle -> Parser s m TaskListItem
+taskItemWithStyle expectedStyle = do
+  item <- taskListItem
+  if taskStyle item == expectedStyle
+    then pure item
+    else fail "Task list marker style mismatch"

src/Utilities/Parsing.hs

@@ -6,11 +6,11 @@ module Utilities.Parsing where
 import Data.String (IsString)
 import Data.Text (Text, pack, unpack)
 import Data.Void (Void)
-import Text.Megaparsec (ParsecT, Stream, Token, Tokens)
+import Text.Megaparsec (ParsecT, Stream, Token, Tokens, TraversableStream)
 
 type Parser = ParsecT Void
 
-class (Token s ~ Char, Stream s, ToText (Tokens s), ToText s, IsString (Tokens s), IsString s, Monoid (Tokens s), ToChar (Token s), Eq (Tokens s), Show s, Monoid s) => Characters s
+class (Token s ~ Char, Stream s, ToText (Tokens s), ToText s, IsString (Tokens s), IsString s, Monoid (Tokens s), ToChar (Token s), Eq (Tokens s), Show s, Monoid s, TraversableStream s) => Characters s
 
 class ToText t where
   toText :: t -> Text