Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
a726753
Add Schema for synced comments
ChrisPenner Nov 20, 2025
3ee61ce
Add new HistoryComments Modules
ChrisPenner Nov 20, 2025
b32c3c9
Write fetch history comments queries
ChrisPenner Nov 20, 2025
2f926b4
Build against new unison
ChrisPenner Nov 20, 2025
d28f2db
Extract streaming utils
ChrisPenner Nov 21, 2025
a843e1c
Skeleton for HistoryComment endpoints
ChrisPenner Nov 21, 2025
4fe9e1e
Write most of the History Comment insert queries
ChrisPenner Nov 21, 2025
23637fb
Use new stream utilities
ChrisPenner Nov 21, 2025
0a0341f
WIP
ChrisPenner Nov 26, 2025
42f929c
WIP
ChrisPenner Nov 26, 2025
229b587
Integrate new Queuing primitives
ChrisPenner Nov 26, 2025
360bf0e
main insert loop done
ChrisPenner Nov 26, 2025
fddf0f1
Do a bunch of error handling
ChrisPenner Nov 26, 2025
081fab1
Add history comments to api
ChrisPenner Nov 26, 2025
9c97a01
Wire up UCM websockets
ChrisPenner Dec 2, 2025
36af719
Wire in API
ChrisPenner Dec 16, 2025
f534112
Fix up thumbprint syncing
ChrisPenner Dec 16, 2025
a48ca15
Make history comment inserts conditional
ChrisPenner Dec 16, 2025
dadab00
Debug errs
ChrisPenner Dec 17, 2025
01480b6
More debugging
ChrisPenner Dec 17, 2025
dde1d87
Fix up query syntax
ChrisPenner Dec 17, 2025
746afb8
Set up comment negotiation system
ChrisPenner Dec 18, 2025
1a99711
Ensure we tell the client when we're done sending hashes
ChrisPenner Jan 5, 2026
9ab7f20
Update sync impl to handle history comments and revisions separately
ChrisPenner Jan 8, 2026
d15f38d
Fix a bunch of comment insertion errors
ChrisPenner Jan 13, 2026
8c3478f
Replace old download auth checks with specific HashJWT overrides
ChrisPenner Jan 13, 2026
a9de783
Add cursor over history comments query functions
ChrisPenner Jan 13, 2026
5193f4c
Write downloading queries
ChrisPenner Jan 13, 2026
4e5f24e
More clean up
ChrisPenner Jan 14, 2026
3901b09
SQL issues
ChrisPenner Jan 14, 2026
442daad
Cleanup
ChrisPenner Jan 14, 2026
946bdf7
Working!
ChrisPenner Jan 14, 2026
499c2dc
Make debugging more specific
ChrisPenner Jan 14, 2026
4c0b80e
Set up history-comments transcripts
ChrisPenner Jan 15, 2026
e9d3451
Remove redundant constraint
ChrisPenner Jan 15, 2026
90eef61
Add comment pull transcripts. Need to update ucm after PRs there are
ChrisPenner Jan 15, 2026
b6b9f91
Rename transcripts
ChrisPenner Jan 20, 2026
5ccee79
Add transcripts checking history comment push/pull
ChrisPenner Jan 20, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,10 @@ jobs:

# Install ucm
mkdir ucm
curl -L https://github.com/unisonweb/unison/releases/download/release%2F1.0.0/ucm-linux-x64.tar.gz | tar -xz -C ucm

# Use latest trunk build to get comment upload/download support for now.
# Old: https://github.com/unisonweb/unison/releases/download/release%2F1.0.0/ucm-linux-x64.tar.gz
curl -L https://github.com/unisonweb/unison/releases/download/trunk-build/ucm-linux-x64.tar.gz | tar -xz -C ucm
export PATH=$PWD/ucm:$PATH

# Start share and it's dependencies in the background
Expand Down
1 change: 1 addition & 0 deletions share-api/package.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ dependencies:
- wai-extra
- wai-middleware-prometheus
- warp
- websockets
- witch
- witherable
- x509
Expand Down
6 changes: 6 additions & 0 deletions share-api/share-api.cabal
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ library
Share.Utils.Servant.Client
Share.Utils.Servant.PathInfo
Share.Utils.Servant.RawRequest
Share.Utils.Servant.Streaming
Share.Utils.Tags
Share.Utils.Unison
Share.Web.Admin.API
Expand Down Expand Up @@ -194,6 +195,9 @@ library
Share.Web.Support.Impl
Share.Web.Support.Types
Share.Web.Types
Share.Web.UCM.HistoryComments.API
Share.Web.UCM.HistoryComments.Impl
Share.Web.UCM.HistoryComments.Queries
Share.Web.UCM.Projects.Impl
Share.Web.UCM.Sync.HashJWT
Share.Web.UCM.Sync.Impl
Expand Down Expand Up @@ -356,6 +360,7 @@ library
, wai-extra
, wai-middleware-prometheus
, warp
, websockets
, witch
, witherable
, x509
Expand Down Expand Up @@ -513,6 +518,7 @@ executable share-api
, wai-extra
, wai-middleware-prometheus
, warp
, websockets
, witch
, witherable
, x509
Expand Down
2 changes: 1 addition & 1 deletion share-api/src/Share/Postgres.hs
Original file line number Diff line number Diff line change
Expand Up @@ -685,7 +685,7 @@ cachedFor = cachedForOf traversed
-- ) SELECT * FROM something JOIN users on something.user_id = users.id
-- |]
-- @@
whenNonEmpty :: forall m f a x. (Monad m, Foldable f, Monoid a) => f x -> m a -> m a
whenNonEmpty :: forall m f a x. (Foldable f, Monoid a, Applicative m) => f x -> m a -> m a
whenNonEmpty f m = if null f then pure mempty else m

timeTransaction :: (QueryM m) => String -> m a -> m a
Expand Down
5 changes: 5 additions & 0 deletions share-api/src/Share/Postgres/IDs.hs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ module Share.Postgres.IDs
NamespaceTermMappingId (..),
NamespaceTypeMappingId (..),
ComponentSummaryDigest (..),
PersonalKeyId (..),

-- * Conversions
hash32AsComponentHash_,
Expand Down Expand Up @@ -104,6 +105,10 @@ newtype ComponentSummaryDigest = ComponentSummaryDigest {unComponentSummaryDiges
deriving stock (Show, Eq, Ord)
deriving (PG.EncodeValue, PG.DecodeValue) via ByteString

newtype PersonalKeyId = PersonalKeyId {unPersonalKeyId :: Int32}
deriving stock (Eq, Ord, Show)
deriving (PG.DecodeValue, PG.EncodeValue) via Int32

toHash32 :: (Coercible h Hash) => h -> Hash32
toHash32 = Hash32.fromHash . coerce

Expand Down
11 changes: 10 additions & 1 deletion share-api/src/Share/Postgres/Orphans.hs
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,11 @@ import U.Codebase.TermEdit qualified as TermEdit
import U.Util.Base32Hex qualified as Base32Hex
import Unison.Hash (Hash)
import Unison.Hash qualified as Hash
import Unison.Hash32 (Hash32)
import Unison.Hash32 (Hash32 (..))
import Unison.Hash32 qualified as Hash32
import Unison.Name (Name)
import Unison.NameSegment.Internal (NameSegment (..))
import Unison.Server.HistoryComments.Types
import Unison.SyncV2.Types (CBORBytes (..))
import Unison.Syntax.Name qualified as Name
import UnliftIO (MonadUnliftIO (..))
Expand Down Expand Up @@ -103,6 +104,14 @@ deriving via Hash instance FromHttpApiData ComponentHash

deriving via Hash instance ToHttpApiData ComponentHash

deriving via Hash32 instance Hasql.DecodeValue HistoryCommentHash32

deriving via Hash32 instance Hasql.EncodeValue HistoryCommentHash32

deriving via Hash32 instance Hasql.DecodeValue HistoryCommentRevisionHash32

deriving via Hash32 instance Hasql.EncodeValue HistoryCommentRevisionHash32

deriving via Text instance Hasql.DecodeValue NameSegment

deriving via Text instance Hasql.EncodeValue NameSegment
Expand Down
4 changes: 4 additions & 0 deletions share-api/src/Share/Prelude/Orphans.hs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
module Share.Prelude.Orphans () where

import Control.Comonad.Cofree (Cofree (..))
import Control.Monad.Except
import Control.Monad.Trans (lift)
import Control.Monad.Trans.Maybe (MaybeT)
import Data.Align (Semialign (..))
Expand Down Expand Up @@ -47,3 +48,6 @@ instance From ShortHash Text where

instance (MonadTracer m) => MonadTracer (MaybeT m) where
getTracer = lift getTracer

instance (MonadTracer m) => MonadTracer (ExceptT e m) where
getTracer = lift getTracer
30 changes: 30 additions & 0 deletions share-api/src/Share/Utils/Logging.hs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import Data.Text qualified as Text
import Data.Text.Encoding qualified as Text
import Data.Text.IO qualified as Text
import GHC.Stack (CallStack, callStack, prettyCallStack)
import Network.WebSockets qualified as WS
import Servant.Client qualified as Servant
import Share.Env.Types qualified as Env
import Share.OAuth.Errors (OAuth2Error)
Expand All @@ -56,6 +57,8 @@ import Share.Utils.Logging.Types as X
import Share.Utils.Tags (MonadTags)
import System.Log.FastLogger qualified as FL
import Unison.Server.Backend qualified as Backend
import Unison.Server.HistoryComments.Types (DownloadCommentsResponse (..), UploadCommentsResponse (..))
import Unison.Server.Types (BranchRef (..))
import Unison.Sync.Types qualified as Sync
import Unison.Util.Monoid (intercalateMap)
import Unison.Util.Monoid qualified as Monoid
Expand Down Expand Up @@ -267,3 +270,30 @@ instance Loggable Sync.UploadEntitiesError where
Sync.UploadEntitiesError'UserNotFound userHandle ->
textLog ("User not found: " <> userHandle)
& withSeverity UserFault

instance Loggable UploadCommentsResponse where
toLog = \case
UploadCommentsProjectBranchNotFound (BranchRef branchRef) ->
textLog ("Project branch not found: " <> branchRef)
& withSeverity UserFault
UploadCommentsNotAuthorized (BranchRef branchRef) ->
textLog ("Not authorized to upload comments to branch: " <> branchRef)
& withSeverity UserFault
UploadCommentsGenericFailure errMsg ->
textLog ("Upload comments generic failure: " <> errMsg)
& withSeverity Error

instance Loggable WS.ConnectionException where
toLog = withSeverity Error . showLog

instance Loggable DownloadCommentsResponse where
toLog = \case
DownloadCommentsProjectBranchNotFound (BranchRef branchRef) ->
textLog ("Project branch not found: " <> branchRef)
& withSeverity UserFault
DownloadCommentsNotAuthorized (BranchRef branchRef) ->
textLog ("Not authorized to download comments from branch: " <> branchRef)
& withSeverity UserFault
DownloadCommentsGenericFailure errMsg ->
textLog ("Download comments generic failure: " <> errMsg)
& withSeverity Error
63 changes: 63 additions & 0 deletions share-api/src/Share/Utils/Servant/Streaming.hs
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
module Share.Utils.Servant.Streaming
( toConduit,
cborStreamToConduit,
fromConduit,
sourceIOWithAsync,
queueToCBORStream,
queueToSourceIO,
)
where

-- Orphan instances for SourceIO

import Codec.Serialise qualified as CBOR
import Conduit
import Control.Concurrent.STM.TBMQueue qualified as STM
import Control.Monad.Except
import Data.ByteString.Builder qualified as Builder
import Ki.Unlifted qualified as Ki
import Servant
import Servant.Conduit (conduitToSourceIO)
import Servant.Types.SourceT
import Share.Prelude
import Unison.Util.Servant.CBOR
import UnliftIO.STM qualified as STM

-- | Run the provided IO action in the background while streaming results.
--
-- Servant doesn't provide any easier way to do bracketing like this, all the IO must be
-- inside the SourceIO somehow.
sourceIOWithAsync :: IO a -> SourceIO r -> SourceIO r
sourceIOWithAsync action (SourceT k) =
SourceT \k' ->
Ki.scoped \scope -> do
_ <- Ki.fork scope action
k k'

toConduit :: (MonadIO m, MonadIO n) => SourceIO o -> m (ConduitT void o n ())
toConduit sourceIO = fmap (transPipe liftIO) . liftIO $ fromSourceIO $ sourceIO

cborStreamToConduit :: (MonadIO m, MonadIO n, CBOR.Serialise o) => SourceIO (CBORStream o) -> m (ConduitT void o (ExceptT CBORStreamError n) ())
cborStreamToConduit sourceIO = toConduit sourceIO <&> \stream -> (stream .| unpackCBORBytesStream)

fromConduit :: ConduitT void o IO () -> SourceIO o
fromConduit = conduitToSourceIO

queueToCBORStream :: forall a f. (CBOR.Serialise a, Foldable f) => STM.TBMQueue (f a) -> ConduitT () (CBORStream a) IO ()
queueToCBORStream q = do
let loop :: ConduitT () (CBORStream a) IO ()
loop = do
liftIO (STM.atomically (STM.readTBMQueue q)) >>= \case
-- The queue is closed.
Nothing -> do
pure ()
Just batches -> do
batches
& foldMap (CBOR.serialiseIncremental)
& (CBORStream . Builder.toLazyByteString)
& Conduit.yield
loop
loop

queueToSourceIO :: forall a f. (CBOR.Serialise a, Foldable f) => STM.TBMQueue (f a) -> SourceIO (CBORStream a)
queueToSourceIO q = fromConduit (queueToCBORStream q)
2 changes: 2 additions & 0 deletions share-api/src/Share/Web/API.hs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import Share.Web.Share.Webhooks.API qualified as Webhooks
import Share.Web.Support.API qualified as Support
import Share.Web.Types
import Share.Web.UCM.SyncV2.API qualified as SyncV2
import Unison.Server.HistoryComments.API qualified as Unison.HistoryComments
import Unison.Share.API.Projects qualified as UCMProjects
import Unison.Sync.API qualified as Unison.Sync

Expand Down Expand Up @@ -53,6 +54,7 @@ type API =
-- This path is deprecated, but is still in use by existing clients.
:<|> ("sync" :> MaybeAuthenticatedSession :> Unison.Sync.API)
:<|> ("ucm" :> "v1" :> "sync" :> MaybeAuthenticatedSession :> Unison.Sync.API)
:<|> ("ucm" :> "v1" :> "history-comments" :> MaybeAuthenticatedUserId :> Unison.HistoryComments.API)
:<|> ("ucm" :> "v1" :> "projects" :> MaybeAuthenticatedSession :> UCMProjects.ProjectsAPI)
:<|> ("ucm" :> "v2" :> "sync" :> MaybeAuthenticatedUserId :> SyncV2.API)
:<|> ("admin" :> Admin.API)
Expand Down
5 changes: 5 additions & 0 deletions share-api/src/Share/Web/Authentication.hs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
module Share.Web.Authentication
( cookieSessionTTL,
requireAuthenticatedUser,
requireAuthenticatedUser',
UnauthenticatedError (..),
pattern MaybeAuthedUserID,
pattern AuthenticatedUser,
Expand Down Expand Up @@ -39,3 +40,7 @@ instance ToServerError UnauthenticatedError where
requireAuthenticatedUser :: Maybe Session -> WebApp UserId
requireAuthenticatedUser (AuthenticatedUser uid) = pure uid
requireAuthenticatedUser _ = Errors.respondError UnauthenticatedError

requireAuthenticatedUser' :: Maybe UserId -> WebApp UserId
requireAuthenticatedUser' (Just uid) = pure uid
requireAuthenticatedUser' _ = Errors.respondError UnauthenticatedError
17 changes: 10 additions & 7 deletions share-api/src/Share/Web/Authorization.hs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ module Share.Web.Authorization
checkUploadToUserCodebase,
checkUploadToProjectBranchCodebase,
checkUserUpdate,
checkDownloadFromUserCodebase,
hashJWTAuthOverride,
checkDownloadFromProjectBranchCodebase,
checkCreateOrg,
checkReadOrgRolesList,
Expand Down Expand Up @@ -389,17 +389,20 @@ checkUploadToUserCodebase reqUserId codebaseOwnerUserId = maybePermissionFailure
assertUsersEqual reqUserId codebaseOwnerUserId
pure $ AuthZ.UnsafeAuthZReceipt Nothing

-- | The download endpoint currently does all of its own auth using HashJWTs,
-- | The download endpoints currently do all of its own auth using HashJWTs,
-- So we don't add any other authz checks here, the HashJWT check is sufficient.
checkDownloadFromUserCodebase :: WebApp (Either AuthZFailure AuthZ.AuthZReceipt)
checkDownloadFromUserCodebase =
hashJWTAuthOverride :: WebApp (Either AuthZFailure AuthZ.AuthZReceipt)
hashJWTAuthOverride =
pure . Right $ AuthZ.UnsafeAuthZReceipt Nothing

-- | The download endpoint currently does all of its own auth using HashJWTs,
-- So we don't add any other authz checks here, the HashJWT check is sufficient.
checkDownloadFromProjectBranchCodebase :: WebApp (Either AuthZFailure AuthZ.AuthZReceipt)
checkDownloadFromProjectBranchCodebase =
pure . Right $ AuthZ.UnsafeAuthZReceipt Nothing
checkDownloadFromProjectBranchCodebase :: Maybe UserId -> ProjectId -> WebApp (Either AuthZFailure AuthZ.AuthZReceipt)
checkDownloadFromProjectBranchCodebase reqUserId projectId =
mapLeft (const authzError) <$> do
checkProjectGet reqUserId projectId
where
authzError = AuthZFailure $ (ProjectPermission (ProjectBranchBrowse projectId))

checkProjectCreate :: UserId -> UserId -> WebApp (Either AuthZFailure AuthZ.AuthZReceipt)
checkProjectCreate reqUserId targetUserId = maybePermissionFailure (ProjectPermission (ProjectCreate targetUserId)) $ do
Expand Down
32 changes: 32 additions & 0 deletions share-api/src/Share/Web/Errors.hs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ import Data.Text qualified as Text
import Data.Text.Encoding qualified as Text
import GHC.Stack qualified as GHC
import GHC.TypeLits qualified as TL
import Network.WebSockets qualified as WS
import Servant
import Servant.Client
import Share.Env.Types qualified as Env
Expand All @@ -67,6 +68,8 @@ import Share.Utils.URI (URIParam (..), addQueryParam)
import Share.Web.App
import Unison.Server.Backend qualified as Backend
import Unison.Server.Errors qualified as Backend
import Unison.Server.HistoryComments.Types (DownloadCommentsResponse (..), UploadCommentsResponse (..))
import Unison.Server.Types (BranchRef (..))
import Unison.Sync.Types qualified as Sync
import UnliftIO qualified

Expand Down Expand Up @@ -423,3 +426,32 @@ instance ToServerError Sync.UploadEntitiesError where
Sync.UploadEntitiesError'NoWritePermission _ -> ("no-write-permission", err403 {errBody = "No Write Permission"})
Sync.UploadEntitiesError'ProjectNotFound _ -> ("project-not-found", err404 {errBody = "Project Not Found"})
Sync.UploadEntitiesError'UserNotFound _ -> ("user-not-found", err404 {errBody = "User Not Found"})

instance ToServerError UploadCommentsResponse where
toServerError = \case
UploadCommentsProjectBranchNotFound (BranchRef branchRef) ->
(ErrorID "upload-comments:project-branch-not-found", err404 {errBody = BL.fromStrict $ Text.encodeUtf8 $ "Project branch not found: " <> branchRef})
UploadCommentsNotAuthorized (BranchRef branchRef) ->
(ErrorID "upload-comments:not-authorized", err403 {errBody = BL.fromStrict $ Text.encodeUtf8 $ "Not authorized to upload comments to branch: " <> branchRef})
UploadCommentsGenericFailure errMsg ->
(ErrorID "upload-comments:generic-failure", err500 {errBody = BL.fromStrict $ Text.encodeUtf8 $ "Upload comments failure: " <> errMsg})

instance ToServerError WS.ConnectionException where
toServerError = \case
WS.CloseRequest _ _ ->
(ErrorID "websocket:close-request", err400 {errBody = "WebSocket closed by client"})
WS.ParseException msg ->
(ErrorID "websocket:parse-exception", err400 {errBody = BL.fromStrict $ Text.encodeUtf8 $ "Invalid message: parse exception: " <> Text.pack msg})
WS.UnicodeException msg ->
(ErrorID "websocket:unicode-exception", err400 {errBody = BL.fromStrict $ Text.encodeUtf8 $ "Unicode decoding exception: " <> Text.pack msg})
WS.ConnectionClosed ->
(ErrorID "websocket:connection-closed", err400 {errBody = "WebSocket connection closed"})

instance ToServerError DownloadCommentsResponse where
toServerError = \case
DownloadCommentsProjectBranchNotFound (BranchRef branchRef) ->
(ErrorID "download-comments:project-branch-not-found", err404 {errBody = BL.fromStrict $ Text.encodeUtf8 $ "Project branch not found: " <> branchRef})
DownloadCommentsNotAuthorized (BranchRef branchRef) ->
(ErrorID "download-comments:not-authorized", err403 {errBody = BL.fromStrict $ Text.encodeUtf8 $ "Not authorized to download comments from branch: " <> branchRef})
DownloadCommentsGenericFailure errMsg ->
(ErrorID "download-comments:generic-failure", err500 {errBody = BL.fromStrict $ Text.encodeUtf8 $ "Download comments failure: " <> errMsg})
2 changes: 2 additions & 0 deletions share-api/src/Share/Web/Impl.hs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import Share.Web.Share.Projects.Impl qualified as Projects
import Share.Web.Share.Webhooks.Impl qualified as Webhooks
import Share.Web.Support.Impl qualified as Support
import Share.Web.Types
import Share.Web.UCM.HistoryComments.Impl qualified as HistoryComments
import Share.Web.UCM.Projects.Impl qualified as UCMProjects
import Share.Web.UCM.Sync.Impl qualified as Sync
import Share.Web.UCM.SyncV2.Impl qualified as SyncV2
Expand Down Expand Up @@ -89,6 +90,7 @@ server =
:<|> healthEndpoint
:<|> Sync.server -- Deprecated path
:<|> Sync.server
:<|> HistoryComments.server
:<|> UCMProjects.server
:<|> SyncV2.server
:<|> Admin.server
Expand Down
9 changes: 9 additions & 0 deletions share-api/src/Share/Web/UCM/HistoryComments/API.hs
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE TypeOperators #-}

module Share.Web.UCM.HistoryComments.API (API) where

import Servant
import Unison.Server.HistoryComments.API qualified as HistoryComments

type API = NamedRoutes HistoryComments.Routes
Loading
Loading