diff --git a/backend/src/api/public/v1/members/createMember.ts b/backend/src/api/public/v1/members/createMember.ts index d112143cc1..49380a5991 100644 --- a/backend/src/api/public/v1/members/createMember.ts +++ b/backend/src/api/public/v1/members/createMember.ts @@ -3,7 +3,7 @@ import { z } from 'zod' import { captureApiChange, memberCreateAction, memberEditIdentitiesAction } from '@crowd/audit-logs' import { getProperDisplayName } from '@crowd/common' -import { insertManyMemberIdentities, createMember as insertMember } from '@crowd/data-access-layer' +import { createMember as insertMember, insertMemberIdentities } from '@crowd/data-access-layer' import { MemberIdentityType } from '@crowd/types' import { optionsQx } from '@/database/sequelizeQueryExecutor' @@ -49,7 +49,7 @@ export async function createMember(req: Request, res: Response): Promise { manuallyCreated: true, }) - const dbIdentities = await insertManyMemberIdentities( + const dbIdentities = await insertMemberIdentities( tx, identities.map((identity) => ({ ...identity, diff --git a/backend/src/api/public/v1/members/identities/createMemberIdentity.ts b/backend/src/api/public/v1/members/identities/createMemberIdentity.ts index bef6c773c6..17d45a9269 100644 --- a/backend/src/api/public/v1/members/identities/createMemberIdentity.ts +++ b/backend/src/api/public/v1/members/identities/createMemberIdentity.ts @@ -7,7 +7,7 @@ import { MemberField, findMemberById, findMemberIdentitiesByValue, - createMemberIdentity as insertMemberIdentity, + insertMemberIdentities, touchMemberUpdatedAt, updateMemberIdentity, } from '@crowd/data-access-layer' @@ -69,20 +69,23 @@ export async function createMemberIdentity(req: Request, res: Response): Promise alreadyExisted = true result = exactMatch } else { - result = await insertMemberIdentity( + const [created] = await insertMemberIdentities( tx, - { - memberId, - platform: data.platform, - value: normalizedValue, - type: data.type, - source: data.source, - verified: data.verified, - verifiedBy: data.verifiedBy, - }, + [ + { + memberId, + platform: data.platform, + value: normalizedValue, + type: data.type, + source: data.source, + verified: data.verified, + verifiedBy: data.verifiedBy, + }, + ], true, true, ) + result = created } // A verified identity confirms the same value for this member, so keep same-value diff --git a/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts b/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts index 1623d4ea0c..dae531bff1 100644 --- a/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts +++ b/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts @@ -85,14 +85,16 @@ export async function patchProjectAffiliation(req: Request, res: Response): Prom if (affiliations.length > 0) { await insertMemberSegmentAffiliations( tx, - memberId, - projectId, affiliations.map((a) => ({ + memberId, + segmentId: projectId, organizationId: a.organizationId, dateStart: a.dateStart.toISOString(), dateEnd: a.dateEnd?.toISOString() ?? null, + verified: true, verifiedBy: verifiedBy!, })), + true, ) } }) diff --git a/backend/src/database/repositories/member/memberAffiliationsRepository.ts b/backend/src/database/repositories/member/memberAffiliationsRepository.ts index 7d4488e957..81a47c421f 100644 --- a/backend/src/database/repositories/member/memberAffiliationsRepository.ts +++ b/backend/src/database/repositories/member/memberAffiliationsRepository.ts @@ -2,7 +2,7 @@ import { OrganizationField, queryOrgs } from '@crowd/data-access-layer' import { deleteMemberSegmentAffiliations, fetchMemberAffiliations, - insertMemberAffiliations, + insertMemberSegmentAffiliations, } from '@crowd/data-access-layer/src/member_segment_affiliations' import { fetchManySegments } from '@crowd/data-access-layer/src/segments' import { IMemberAffiliation, IOrganization, SegmentData } from '@crowd/types' @@ -89,8 +89,17 @@ class MemberAffiliationsRepository { await deleteMemberSegmentAffiliations(qx, { memberId }) if (data?.length > 0) { - // Insert multiple member affiliations - await insertMemberAffiliations(qx, memberId, data) + await insertMemberSegmentAffiliations( + qx, + data.map((item) => ({ + memberId, + segmentId: item.segmentId, + organizationId: item.organizationId, + dateStart: item.dateStart || null, + dateEnd: item.dateEnd || null, + })), + true, + ) } await SequelizeRepository.commitTransaction(transaction) diff --git a/backend/src/database/repositories/memberRepository.ts b/backend/src/database/repositories/memberRepository.ts index 3cc917b0e0..8d4af4f24a 100644 --- a/backend/src/database/repositories/memberRepository.ts +++ b/backend/src/database/repositories/memberRepository.ts @@ -18,11 +18,11 @@ import { import { BotDetectionService, CommonMemberService } from '@crowd/common_services' import { OrganizationField, - createMemberIdentity, deleteMemberIdentities, deleteMemberIdentitiesByCombinations, findAlreadyExistingVerifiedIdentities, getLastActivitiesForMembers, + insertMemberIdentities, queryActivityRelations, queryOrgs, updateVerifiedFlag, @@ -33,7 +33,7 @@ import { addMemberNoMerge, removeMemberToMerge } from '@crowd/data-access-layer/ import { deleteMemberSegmentAffiliations, findMemberAffiliations, - insertMemberAffiliations, + insertMemberSegmentAffiliations, } from '@crowd/data-access-layer/src/member_segment_affiliations' import { MemberField, @@ -159,8 +159,9 @@ class MemberRepository { const subprojectIds = await getSegmentSubprojectIds(qx, currentSegments) if (data.identities) { - for (const i of data.identities as IMemberIdentity[]) { - await createMemberIdentity(qx, { + await insertMemberIdentities( + qx, + (data.identities as IMemberIdentity[]).map((i) => ({ memberId: record.id, platform: i.platform, type: i.type, @@ -169,15 +170,16 @@ class MemberRepository { integrationId: i.integrationId || null, verified: i.verified, source: i.source, - }) - } + })), + ) } else if (data.username) { const username: PlatformIdentities = mapUsernameToIdentities(data.username) + const identitiesToInsert = [] for (const platform of Object.keys(username) as PlatformType[]) { const identities: any[] = username[platform] for (const identity of identities) { - await createMemberIdentity(qx, { + identitiesToInsert.push({ memberId: record.id, platform, value: identity.value ? identity.value : identity.username, @@ -189,6 +191,10 @@ class MemberRepository { }) } } + + if (identitiesToInsert.length > 0) { + await insertMemberIdentities(qx, identitiesToInsert) + } } await includeMemberToSegments(qx, record.id, subprojectIds) @@ -1049,8 +1055,9 @@ class MemberRepository { } if (data.identitiesToCreate && data.identitiesToCreate.length > 0) { - for (const i of data.identitiesToCreate) { - await createMemberIdentity(qx, { + await insertMemberIdentities( + qx, + data.identitiesToCreate.map((i) => ({ memberId: record.id, platform: i.platform, value: i.value, @@ -1059,8 +1066,8 @@ class MemberRepository { integrationId: i.integrationId || null, verified: i.verified !== undefined ? i.verified : !!manualChange, source: i.source, - }) - } + })), + ) } if (data.identitiesToUpdate && data.identitiesToUpdate.length > 0) { @@ -1094,6 +1101,7 @@ class MemberRepository { const platformsToDelete: string[] = [] const valuesToDelete: string[] = [] const typesToDelete: MemberIdentityType[] = [] + const identitiesToInsert = [] for (const platform of platforms) { const identities = data.username[platform] @@ -1112,7 +1120,7 @@ class MemberRepository { (identity.username && identity.username !== '') || (identity.value && identity.value !== '') ) { - await createMemberIdentity(qx, { + identitiesToInsert.push({ memberId: record.id, platform, value: identity.value ? identity.value : identity.username, @@ -1126,6 +1134,10 @@ class MemberRepository { } } + if (identitiesToInsert.length > 0) { + await insertMemberIdentities(qx, identitiesToInsert) + } + if (platformsToDelete.length > 0) { await deleteMemberIdentitiesByCombinations(qx, { memberId: record.id, @@ -1207,7 +1219,17 @@ class MemberRepository { return } - await insertMemberAffiliations(qx, memberId, data) + await insertMemberSegmentAffiliations( + qx, + data.map((item) => ({ + memberId, + segmentId: item.segmentId, + organizationId: item.organizationId, + dateStart: item.dateStart || null, + dateEnd: item.dateEnd || null, + })), + true, + ) }), ) } diff --git a/backend/src/database/repositories/organizationRepository.ts b/backend/src/database/repositories/organizationRepository.ts index 086c3edb50..658dc29b14 100644 --- a/backend/src/database/repositories/organizationRepository.ts +++ b/backend/src/database/repositories/organizationRepository.ts @@ -16,7 +16,6 @@ import { IDbOrganization, OrgIdentityField, OrganizationField, - addOrgIdentity, addOrgsToSegments, cleanUpOrgIdentities, cleanupForOganization, @@ -27,6 +26,7 @@ import { findManyOrgAttributes, findOrgAttributes, findOrgById, + insertOrganizationIdentities, markOrgAttributeDefault, queryOrgIdentities, updateOrgIdentityVerifiedFlag, @@ -619,16 +619,22 @@ class OrganizationRepository { ): Promise { const qx = SequelizeRepository.getQueryExecutor(options) - await addOrgIdentity(qx, { - organizationId, - platform: identity.platform, - source: identity.source, - sourceId: identity.sourceId || null, - value: identity.value, - type: identity.type, - verified: identity.verified, - integrationId: identity.integrationId || null, - }) + await insertOrganizationIdentities( + qx, + [ + { + organizationId, + platform: identity.platform, + source: identity.source, + sourceId: identity.sourceId || null, + value: identity.value, + type: identity.type, + verified: identity.verified, + integrationId: identity.integrationId || null, + }, + ], + false, + ) } static async getIdentities( diff --git a/backend/src/services/member/memberIdentityService.ts b/backend/src/services/member/memberIdentityService.ts index 48569bf93b..0aa85205aa 100644 --- a/backend/src/services/member/memberIdentityService.ts +++ b/backend/src/services/member/memberIdentityService.ts @@ -3,7 +3,7 @@ import lodash from 'lodash' import { captureApiChange, memberEditIdentitiesAction } from '@crowd/audit-logs' import { Error404, Error409 } from '@crowd/common' -import { createMemberIdentity, findIdentitiesForMembers } from '@crowd/data-access-layer' +import { findIdentitiesForMembers, insertMemberIdentities } from '@crowd/data-access-layer' import { deleteMemberIdentity, fetchMemberIdentities, @@ -77,7 +77,7 @@ export default class MemberIdentityService extends LoggerBase { } // Create member identity - await createMemberIdentity(qx, { ...data, memberId }) + await insertMemberIdentities(qx, [{ ...data, memberId }]) await touchMemberUpdatedAt(qx, memberId) @@ -151,9 +151,10 @@ export default class MemberIdentityService extends LoggerBase { } // Create member identities - for (const identity of data) { - await createMemberIdentity(qx, { ...identity, memberId }) - } + await insertMemberIdentities( + qx, + data.map((identity) => ({ ...identity, memberId })), + ) await touchMemberUpdatedAt(qx, memberId) diff --git a/docs/adr/0006-database-schema-types-as-source-of-truth.md b/docs/adr/0006-database-schema-types-as-source-of-truth.md new file mode 100644 index 0000000000..1946afbc53 --- /dev/null +++ b/docs/adr/0006-database-schema-types-as-source-of-truth.md @@ -0,0 +1,61 @@ +# ADR-0006: Database schema types as the source of truth + +**Date**: 2026-07-09 +**Status**: accepted +**Deciders**: Yeganathan S + +## Context + +Working with entity types in CDP is messy. The same concepts appear as overlapping shapes across `@crowd/types`, DAL inputs, merge helpers, and API/domain models — often incomplete relative to the database, with inconsistent nullability and no single source of truth. That makes it hard to change data access safely, reuse types across packages, and keep tests aligned with production. + +We need a durable way to model persisted data that the rest of the codebase can build on, without inventing yet another parallel type family per feature. + +## Decision + +Treat the database schema as the source of truth for persisted entity shapes. Introduce a dedicated `db/` area under `@crowd/types` for schema-aligned row types, and derive narrower types (create/update payloads, factory options, domain views) from those via composition (`Pick`, `Partial`, `Omit`, intersections) rather than hand-maintaining duplicates. + +These types live in `@crowd/types`, not in the data-access layer: they are shared vocabulary for backend, workers, libraries, and test utilities. DAL remains responsible for queries and persistence; it consumes and returns these types instead of owning the canonical shapes. + +This applies to entities generally (members, identities, organizations, and others as we touch them), not only the first tables we migrate. + +## Conventions + +These rules exist so people (and tools) do not confuse the row type with write payloads, or nullability with “field may be omitted.” + +1. **One row type per table** — describes the entity as stored. Field nullability matches the schema. Database defaults do not make read fields optional. +2. **Write payloads are derived from the row type** — use `Pick`, `Partial`, `Omit`, and intersections instead of duplicating the full interface. +3. **Nullability ≠ optionality** — `| null` means the column can store null; `?` / `Partial` means the caller may omit the key. Both can appear; they mean different things. +4. **System-owned fields stay off write payloads** — things like tenant id, audit timestamps, and soft-delete markers are set by persistence code, not by every caller. +5. **Layers that supply defaults may widen further** — e.g. a test factory can take a partial write payload and fill required fields before calling DAL. That does not change the row type or the DAL contract. + +## Alternatives Considered + +### Alternative 1: Keep defining types ad hoc next to each feature +- **Pros**: Fast locally; no cross-package coordination. +- **Cons**: Duplicates and drift continue; nullability and field sets disagree across call sites. +- **Why not**: That is the current pain. It does not scale as more packages share the same entities. + +### Alternative 2: Own canonical entity types inside `@crowd/data-access-layer` +- **Pros**: Types sit next to SQL; easy for DAL authors to update. +- **Cons**: Anything that needs a row shape (test-kit, common services, workers, backend) must depend on DAL or re-copy types. +- **Why not**: Row shapes are shared contracts, not DAL implementation details. Putting them in `@crowd/types` keeps the dependency graph thin and matches how other shared models already live. + +### Alternative 3: Generate types from the schema (e.g. introspection tooling) +- **Pros**: Always in sync with migrations; less manual work. +- **Cons**: Tooling and review process are not in place; generated output can be noisy and hard to evolve with domain naming. +- **Why not**: Manual schema-aligned types are enough to establish the pattern now. Generation can be revisited once the convention is stable. + +## Consequences + +### Positive +- One place to look for “what does this table look like in TypeScript.” +- Downstream APIs and tests extend from the same base instead of inventing parallel models. +- Clearer package boundaries: `@crowd/types` for shapes, DAL for access, factories/services for composition. + +### Negative +- Existing aliases and legacy shapes (`MemberRow`, feature-local inputs, etc.) will coexist during migration. +- Authors must update `db/` types when migrations change columns or nullability. + +### Risks +- **Partial adoption** — mitigated by migrating types when a table is touched (factories, DAL hardening, new features), not a big-bang rewrite. +- **Drift from schema** — mitigated by checking live schema (or migrations) when adding or changing `db/` types; optional codegen later. diff --git a/docs/adr/0007-test-factory-primitives-and-defaults.md b/docs/adr/0007-test-factory-primitives-and-defaults.md new file mode 100644 index 0000000000..9e9de8336b --- /dev/null +++ b/docs/adr/0007-test-factory-primitives-and-defaults.md @@ -0,0 +1,83 @@ +# ADR-0007: Test factory primitives and defaults + +**Date**: 2026-07-10 +**Status**: accepted +**Deciders**: Yeganathan S + +## Context + +Shared test factories are easy to turn into hidden scenario builders: they invent data the test never declared, blur what is under assertion, and drift from the real insert shapes used in production code (see ADR-0006). + +We need a simple, durable rule for how test data is created — what the factory does, what defaults may fill, and what the test must own — so the pattern does not diverge as more factories are added. + +## Decision + +Treat test setup as composition of sharp primitives, not smart world-builders. Primitives persist what the caller provides via the DAL; defaults are opt-in and allowlisted; tests own scenario identity and graph shape. + +## Conventions + +### 1. Primitives + +- One job: **persist what they are given**, preferably in bulk, via the DAL. +- Input stays close to schema insert types (`*DbInsert` and small, explicit extensions when a factory accepts related children in one call). +- No faker inside primitives. No silent graph scaffolding. No behavioral guesses. +- When a primitive accepts structured input (for example a hierarchy), it may **derive structural fields** from that shape (links, type/level, denormalized parent labels). That derivation is part of the primitive’s contract — not a default. + +### 2. Defaults (opt-in) + +- Separate helpers (`withXDefaults`) fill an **allowlist** of missing label / harmless system fields **on rows the caller already wrote**. +- Call sites opt in: `createX(qx, withXDefaults([...]))`. Defaults are never baked into primitives. +- Semantics: + - **Allowlist only** — unlisted fields are untouched. + - **Explicit always wins** — a provided value is never overwritten. + - **Only** `undefined` **is missing** — omitted / `undefined` may be filled; `null` **is intentional and sticks**. + - **Labels / harmless flags only** — e.g. generated `id`, display names, join timestamps, inactive-path status flags. Ask per field: *if we invent this, could we change which production branch the test hits?* If yes, it does not belong in defaults. + - **Caller owns scenario identity** when the value defines the case (names, slugs, platforms, dates, relationship targets, and any field that makes two cases different). + +Defaults do **not** build a realistic entity or graph. They only patch allowlisted gaps (e.g. `displayName`, `joinedAt`). Identities, org stints, activities, and other scenario data stay with the test. + +### 3. Test composition + +- The suite owns behavioral fields and graph shape. +- Prefer inline fixtures and spreads in the test file. +- File-local seed helpers only after real repetition; promote into the shared test kit only when a second suite needs the same helper. +- Do not grow suite-specific “build the whole world” APIs in the shared kit. + +## Alternatives Considered + +### Alternative 1: Always-on smart factories that scaffold graphs + +- **Pros**: Shortest test setup; one call builds a “ready” world. +- **Cons**: Hides scenario data; easy to pass for the wrong reason; hard to see what a test asserts. +- **Why not**: Tests must declare behavior. Defaults are ergonomics, not fixtures. + +### Alternative 2: No shared defaults — every test passes full insert rows + +- **Pros**: Maximum explicitness; zero magic. +- **Cons**: Noisy boilerplate for ids/labels/flags that rarely matter to the assertion. +- **Why not**: Opt-in allowlisted defaults keep noise down without hiding scenario data. + +### Alternative 3: Defaults baked into every primitive call + +- **Pros**: Slightly shorter call sites. +- **Cons**: Forces defaults even when a test wants raw inserts; harder to see the boundary between persistence and convenience. +- **Why not**: Opt-in composition keeps primitives sharp and defaults obvious at the call site. + +## Consequences + +### Positive + +- Factories stay readable, composable, and reviewable; scenario intent stays in the test. +- Same overwrite semantics everywhere (`undefined` fill, `null` preserved, allowlist only). +- New factories have a clear checklist instead of reinventing “helpful” behavior. +- Stays aligned with ADR-0006’s insert/row types. + +### Negative + +- Callers must compose `withXDefaults` explicitly. +- Suites carry more scenario data in the test file than a mega-helper would. + +### Risks + +- **Defaults creep** — mitigate by reviewing allowlists against “could this change the branch we hit?” and rejecting behavioral defaults in review. +- **Scenario helpers leaking into the shared kit** — mitigate by keeping suite-specific seeds file-local until a second suite needs them. diff --git a/docs/adr/README.md b/docs/adr/README.md index 250066a4a4..839884caed 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -12,6 +12,8 @@ Use the `/adr` skill in Claude Code to record new ADRs or query past decisions. | [ADR-0003](./0003-deps-bq-table-selection.md) | Use DependencyGraphEdgesLatest for deps ingestion; defer DependenciesLatest until NUGET or GO needed | accepted | 2026-05-29 | | [ADR-0004](./0004-go-nuget-transitive-dependent-counts.md) | Compute GO/NUGET transitive dependent counts via exact reverse closure (over HLL approximation) | accepted | 2026-06-23 | | [ADR-0005](./0005-pypi-downloads-bigquery-merge-scoping.md) | PyPI downloads via BigQuery bulk export, scoped in the Postgres merge | accepted | 2026-07-01 | +| [ADR-0006](./0006-database-schema-types-as-source-of-truth.md) | Database schema types as the source of truth | accepted | 2026-07-09 | +| [ADR-0007](./0007-test-factory-primitives-and-defaults.md) | Test factory primitives and defaults | accepted | 2026-07-10 | ## Why ADRs? diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9b2994d141..76c5d48ea1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2265,12 +2265,18 @@ importers: specifier: ^13.7.0 version: 13.11.0 devDependencies: + '@crowd/test-kit': + specifier: workspace:* + version: link:../test-kit '@types/node': specifier: ^20.8.2 version: 20.12.7 typescript: specifier: ^5.6.3 version: 5.6.3 + vitest: + specifier: 4.1.7 + version: 4.1.7(@types/node@20.12.7)(vite@6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0)) services/libs/database: dependencies: @@ -2594,9 +2600,21 @@ importers: '@crowd/common': specifier: workspace:* version: link:../common + '@crowd/data-access-layer': + specifier: workspace:* + version: link:../data-access-layer '@crowd/database': specifier: workspace:* version: link:../database + '@crowd/integrations': + specifier: workspace:* + version: link:../integrations + '@crowd/types': + specifier: workspace:* + version: link:../types + '@faker-js/faker': + specifier: ^9.9.0 + version: 9.9.0 pg-promise: specifier: ^11.4.3 version: 11.6.0 @@ -3635,6 +3653,10 @@ packages: '@exodus/schemasafe@1.3.0': resolution: {integrity: sha512-5Aap/GaRupgNx/feGBwLLTVv8OQFfv3pq2lPRzPg9R+IOBnDgghTGW7l7EuVXOvg5cc/xSAlRW8rBrjIC3Nvqw==} + '@faker-js/faker@9.9.0': + resolution: {integrity: sha512-OEl393iCOoo/z8bMezRlJu+GlRGlsKbUAN7jKB6LhnKoqKve5DXRpalbItIIcwnCjs1k/FOPjFzcA6Qn+H+YbA==} + engines: {node: '>=18.0.0', npm: '>=9.0.0'} + '@fastify/busboy@2.1.1': resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} engines: {node: '>=14'} @@ -12424,6 +12446,8 @@ snapshots: '@exodus/schemasafe@1.3.0': {} + '@faker-js/faker@9.9.0': {} + '@fastify/busboy@2.1.1': {} '@gitbeaker/core@43.5.0': @@ -19201,7 +19225,7 @@ snapshots: dependencies: '@types/debug': 4.1.12 '@types/validator': 13.11.9 - debug: 4.4.0(supports-color@5.5.0) + debug: 4.4.3 dottie: 2.0.6 inflection: 1.13.4 lodash: 4.17.21 diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts index 726375f659..1ffb8ffbc3 100644 --- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts +++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts @@ -13,11 +13,11 @@ import { signalMemberUpdate } from '@crowd/common_services' import { changeMemberOrganizationAffiliationOverrides, fetchManyOrganizationAffiliationPolicies, + insertMemberIdentities, updateMemberAttributes, updateMemberContributions, updateMemberReach, } from '@crowd/data-access-layer' -import { createMemberIdentity } from '@crowd/data-access-layer' import { findMemberIdentityWithTheMostActivityInPlatform as getMemberMostActiveIdentity } from '@crowd/data-access-layer/src/activityRelations' import { deleteMemberSegmentAffiliations } from '@crowd/data-access-layer/src/member_segment_affiliations' import { getPlatformPriorityArray } from '@crowd/data-access-layer/src/members/attributeSettings' @@ -36,9 +36,9 @@ import { } from '@crowd/data-access-layer/src/old/apps/members_enrichment_worker' import OrganizationMergeSuggestionsRepository from '@crowd/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo' import { - addOrgIdentity, findOrCreateOrganization, findOrgByVerifiedIdentity, + insertOrganizationIdentities, } from '@crowd/data-access-layer/src/organizations' import { dbStoreQx, pgpQx } from '@crowd/data-access-layer/src/queryExecutor' import { refreshMaterializedView } from '@crowd/data-access-layer/src/utils' @@ -308,21 +308,19 @@ export async function updateMemberUsingSquashedPayload( // process identities if (squashedPayload.identities.length > 0) { svc.log.debug({ memberId }, 'Adding to member identities!') - for (const i of squashedPayload.identities) { - didUpdate = true - await createMemberIdentity( - qx, - { - memberId, - platform: i.platform, - type: i.type, - value: i.value, - verified: i.verified, - source: 'enrichment', - }, - true, - ) - } + didUpdate = true + await insertMemberIdentities( + qx, + squashedPayload.identities.map((i) => ({ + memberId, + platform: i.platform, + type: i.type, + value: i.value, + verified: i.verified, + source: 'enrichment', + })), + true, + ) } // process contributions @@ -518,17 +516,23 @@ export async function updateMemberUsingSquashedPayload( const mergeSuggestions = [] const suggestedOwnerIds = new Set() + const identitiesToInsert = identityOwners + .filter((identityOwner) => identityOwner.organizationId !== orgId) + .map((identityOwner) => ({ + organizationId: orgId, + platform: identityOwner.identity.platform, + value: identityOwner.identity.value, + type: identityOwner.identity.type, + verified: false, + source: orgSource, + })) + + if (identitiesToInsert.length > 0) { + await insertOrganizationIdentities(qx, identitiesToInsert, false) + } + for (const identityOwner of identityOwners) { if (identityOwner.organizationId !== orgId) { - await addOrgIdentity(qx, { - organizationId: orgId, - platform: identityOwner.identity.platform, - value: identityOwner.identity.value, - type: identityOwner.identity.type, - verified: false, - source: orgSource, - }) - const noMergeIds = await mergeSuggestionsRepo.findNoMergeIds( identityOwner.organizationId, ) diff --git a/services/apps/members_enrichment_worker/src/activities/member.ts b/services/apps/members_enrichment_worker/src/activities/member.ts index f34855c343..ccf30f34ff 100644 --- a/services/apps/members_enrichment_worker/src/activities/member.ts +++ b/services/apps/members_enrichment_worker/src/activities/member.ts @@ -2,8 +2,8 @@ import { CommonMemberService } from '@crowd/common_services' import { MemberField, PgPromiseQueryExecutor, - createMemberIdentity, findMemberById, + insertMemberIdentities, pgpQx, } from '@crowd/data-access-layer' import { @@ -88,15 +88,18 @@ export async function updateMemberWithEnrichmentData( attributes?: IAttributes, ): Promise { await svc.postgres.writer.connection().tx(async (tx) => { - for (const identity of identities) { - await createMemberIdentity(new PgPromiseQueryExecutor(tx), { - memberId, - platform: identity.platform, - value: identity.value, - type: identity.type, - verified: identity.verified || false, - source: 'enrichment', - }) + if (identities.length > 0) { + await insertMemberIdentities( + new PgPromiseQueryExecutor(tx), + identities.map((identity) => ({ + memberId, + platform: identity.platform, + value: identity.value, + type: identity.type, + verified: identity.verified || false, + source: 'enrichment', + })), + ) } if (attributes) { await updateMemberAttributes(tx, memberId, attributes) diff --git a/services/libs/common_services/src/services/member/unmerge.ts b/services/libs/common_services/src/services/member/unmerge.ts index fdba875892..73c3ae9878 100644 --- a/services/libs/common_services/src/services/member/unmerge.ts +++ b/services/libs/common_services/src/services/member/unmerge.ts @@ -10,7 +10,6 @@ import { addMemberRole, changeMemberOrganizationAffiliationOverrides, createMember, - createMemberIdentity, deleteManyMemberIdentities, fetchManyMemberOrgsWithOrgData, fetchManyOrganizationAffiliationPolicies, @@ -18,6 +17,7 @@ import { findAlreadyExistingVerifiedIdentities, findMemberById, findNonExistingOrganizationIds, + insertMemberIdentities, removeMemberRole, updateMember, } from '@crowd/data-access-layer' @@ -502,17 +502,20 @@ export async function unmergeMember( ) // Create identities for the secondary member - for (const i of secondaryIdentities) { - await createMemberIdentity(tx, { - memberId: secondaryId, - platform: i.platform, - type: i.type, - value: i.value, - sourceId: i.sourceId || null, - integrationId: i.integrationId || null, - verified: i.verified, - source: i.source, - }) + if (secondaryIdentities.length > 0) { + await insertMemberIdentities( + tx, + secondaryIdentities.map((i) => ({ + memberId: secondaryId, + platform: i.platform, + type: i.type, + value: i.value, + sourceId: i.sourceId || null, + integrationId: i.integrationId || null, + verified: i.verified, + source: i.source, + })), + ) } // Move affiliations diff --git a/services/libs/data-access-layer/package.json b/services/libs/data-access-layer/package.json index 16579c5b0b..dbdfb90724 100644 --- a/services/libs/data-access-layer/package.json +++ b/services/libs/data-access-layer/package.json @@ -33,7 +33,9 @@ "validator": "^13.7.0" }, "devDependencies": { + "@crowd/test-kit": "workspace:*", "@types/node": "^20.8.2", - "typescript": "^5.6.3" + "typescript": "^5.6.3", + "vitest": "4.1.7" } } diff --git a/services/libs/data-access-layer/src/activities/sql.ts b/services/libs/data-access-layer/src/activities/sql.ts index 54cbac19e8..0887cb7e7b 100644 --- a/services/libs/data-access-layer/src/activities/sql.ts +++ b/services/libs/data-access-layer/src/activities/sql.ts @@ -9,7 +9,7 @@ import { Counter, TinybirdClient, } from '@crowd/database' -import { ActivityDisplayService } from '@crowd/integrations' +import { ActivityDisplayService } from '@crowd/integrations/src/integrations/activityDisplayService' import { ActivityTypeSettings, ITimeseriesDatapoint, PageData } from '@crowd/types' import { getLatestMemberActivityRelations } from '../activityRelations' diff --git a/services/libs/data-access-layer/src/index.ts b/services/libs/data-access-layer/src/index.ts index 9ede0d95cd..9079622755 100644 --- a/services/libs/data-access-layer/src/index.ts +++ b/services/libs/data-access-layer/src/index.ts @@ -6,10 +6,12 @@ export * from './dashboards' export * from './members' export * from './organizations' export * from './member-organization-affiliation' +export * from './member_segment_affiliations' export * from './prompt-history' export * from './queryExecutor' export * from './repositories' export * from './security_insights' +export * from './segments' export * from './systemSettings' export * from './integrations' export * from './auditLogs' diff --git a/services/libs/data-access-layer/src/member-organization-affiliation/index.ts b/services/libs/data-access-layer/src/member-organization-affiliation/index.ts index 0e3f3115bb..a81c54c3ae 100644 --- a/services/libs/data-access-layer/src/member-organization-affiliation/index.ts +++ b/services/libs/data-access-layer/src/member-organization-affiliation/index.ts @@ -7,6 +7,7 @@ import { IChangeAffiliationOverrideData, IMemberOrganization, IMemberOrganizationAffiliationOverride, + type MemberOrganizationAffiliationOverrideDbRow, } from '@crowd/types' import { findMemberAffiliations } from '../member_segment_affiliations' @@ -31,7 +32,7 @@ const isMemberOrganizationWithOverrides = ( row: AffiliationItem, ): row is MemberOrganizationWithOverrides => !isManualAffiliation(row) -async function prepareMemberOrganizationAffiliationTimeline( +export async function prepareMemberOrganizationAffiliationTimeline( qx: QueryExecutor, memberId: string, ): Promise { @@ -522,9 +523,20 @@ export async function refreshMemberOrganizationAffiliations(qx: QueryExecutor, m export async function changeMemberOrganizationAffiliationOverrides( qx: QueryExecutor, data: IChangeAffiliationOverrideData[], -): Promise { + returnRows: true, +): Promise +export async function changeMemberOrganizationAffiliationOverrides( + qx: QueryExecutor, + data: IChangeAffiliationOverrideData[], + returnRows?: false, +): Promise +export async function changeMemberOrganizationAffiliationOverrides( + qx: QueryExecutor, + data: IChangeAffiliationOverrideData[], + returnRows = false, +): Promise { if (!Array.isArray(data) || data.length === 0) { - return + return returnRows ? [] : undefined } const rows: IMemberOrganizationAffiliationOverride[] = [] @@ -539,7 +551,7 @@ export async function changeMemberOrganizationAffiliationOverrides( } rows.push({ - id: uuid(), + id: d.id ?? uuid(), memberId: d.memberId, memberOrganizationId: d.memberOrganizationId, allowAffiliation: d.allowAffiliation, @@ -548,7 +560,7 @@ export async function changeMemberOrganizationAffiliationOverrides( } if (rows.length === 0) { - return + return returnRows ? [] : undefined } const valuesSql = rows @@ -577,8 +589,7 @@ export async function changeMemberOrganizationAffiliationOverrides( {} as Record, ) - await qx.result( - ` + const query = ` INSERT INTO "memberOrganizationAffiliationOverrides" ( id, "memberId", @@ -590,10 +601,15 @@ export async function changeMemberOrganizationAffiliationOverrides( ON CONFLICT ("memberId", "memberOrganizationId") DO UPDATE SET "allowAffiliation" = COALESCE(EXCLUDED."allowAffiliation", "memberOrganizationAffiliationOverrides"."allowAffiliation"), - "isPrimaryWorkExperience" = COALESCE(EXCLUDED."isPrimaryWorkExperience", "memberOrganizationAffiliationOverrides"."isPrimaryWorkExperience"); - `, - params, - ) + "isPrimaryWorkExperience" = COALESCE(EXCLUDED."isPrimaryWorkExperience", "memberOrganizationAffiliationOverrides"."isPrimaryWorkExperience") + ${returnRows ? 'RETURNING *' : ''} + ` + + if (returnRows) { + return qx.select(query, params) + } + + await qx.result(query, params) } export async function findMemberAffiliationOverrides( diff --git a/services/libs/data-access-layer/src/member_segment_affiliations/index.ts b/services/libs/data-access-layer/src/member_segment_affiliations/index.ts index 255b6077c4..91ed33a4af 100644 --- a/services/libs/data-access-layer/src/member_segment_affiliations/index.ts +++ b/services/libs/data-access-layer/src/member_segment_affiliations/index.ts @@ -1,6 +1,9 @@ -import { v4 as uuid } from 'uuid' - -import { IMemberAffiliation } from '@crowd/types' +import { generateUUIDv1 } from '@crowd/common' +import { + type IMemberAffiliation, + type MemberSegmentAffiliationDbInsert, + type MemberSegmentAffiliationDbRow, +} from '@crowd/types' import { IManualAffiliationData } from '../old/apps/data_sink_worker/repo/memberAffiliation.data' import { QueryExecutor } from '../queryExecutor' @@ -78,22 +81,54 @@ export async function findMemberAffiliations( ) } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export async function insertMemberAffiliations(qx: QueryExecutor, memberId: string, data: any[]) { - return qx.result( - prepareBulkInsert( - 'memberSegmentAffiliations', - ['id', 'memberId', 'segmentId', 'organizationId', 'dateStart', 'dateEnd'], - data.map((item) => ({ - id: uuid(), - memberId, - segmentId: item.segmentId, - organizationId: item.organizationId, - dateStart: item.dateStart || null, - dateEnd: item.dateEnd || null, - })), - ), +export async function insertMemberSegmentAffiliations( + qx: QueryExecutor, + affiliations: MemberSegmentAffiliationDbInsert[], + failOnConflict: boolean, + returnRows: true, +): Promise +export async function insertMemberSegmentAffiliations( + qx: QueryExecutor, + affiliations: MemberSegmentAffiliationDbInsert[], + failOnConflict?: boolean, + returnRows?: false, +): Promise +export async function insertMemberSegmentAffiliations( + qx: QueryExecutor, + affiliations: MemberSegmentAffiliationDbInsert[], + failOnConflict = false, + returnRows = false, +): Promise { + if (affiliations.length === 0) { + return returnRows ? [] : 0 + } + + const query = prepareBulkInsert( + 'memberSegmentAffiliations', + [ + 'id', + 'memberId', + 'segmentId', + 'organizationId', + 'dateStart', + 'dateEnd', + 'verified', + 'verifiedBy', + ], + affiliations.map((a) => ({ + ...a, + id: a.id ?? generateUUIDv1(), + verified: a.verified ?? false, + })), + failOnConflict ? undefined : 'DO NOTHING', + returnRows, ) + + if (returnRows) { + return qx.select(query) + } + + return qx.result(query) } export async function fetchMemberAffiliations( diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 767db22dcc..eb5fbd37c3 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -15,6 +15,8 @@ import { ALL_PLATFORM_TYPES, IMemberContribution, MemberAttributeType, + MemberDbInsert, + MemberDbRow, MemberRow, PageData, SegmentType, @@ -75,15 +77,8 @@ export enum MemberField { UPDATED_BY_ID = 'updatedById', } -export interface MemberCreateInput { - id?: string - displayName: string - joinedAt: string - attributes: Record - reach: Partial> - manuallyCreated?: boolean - contributions?: IMemberContribution[] -} +/** @deprecated Prefer `MemberDbInsert` from `@crowd/types`. */ +export type MemberCreateInput = MemberDbInsert export interface MemberUpdateInput { joinedAt?: string @@ -132,13 +127,19 @@ export const MEMBER_INSERT_COLUMNS = [ 'attributes', 'contributions', 'createdAt', + 'createdById', 'displayName', + 'enrichedBy', 'id', + 'importHash', 'joinedAt', + 'manuallyChangedFields', 'manuallyCreated', 'reach', + 'score', 'tenantId', 'updatedAt', + 'updatedById', ] const QUERY_FILTER_COLUMN_MAP: Map = new Map([ @@ -752,15 +753,10 @@ export async function updateMember( return qx.selectOneOrNone(`${query} ${condition} returning *`) } -export async function createMember(qx: QueryExecutor, data: MemberCreateInput): Promise { +export async function createMember(qx: QueryExecutor, data: MemberDbInsert): Promise { const id = data.id ?? generateUUIDv1() const ts = new Date() const dbInstance = getDbInstance() - const columnSet = new dbInstance.helpers.ColumnSet(MEMBER_INSERT_COLUMNS, { - table: { - table: 'members', - }, - }) const dbData: Record = { ...data, @@ -771,6 +767,16 @@ export async function createMember(qx: QueryExecutor, data: MemberCreateInput): updatedAt: ts, } + // Omit unset columns so Postgres DEFAULT applies (listing a column with + // undefined/null bypasses DEFAULT and inserts NULL instead). + const columns = MEMBER_INSERT_COLUMNS.filter((column) => dbData[column] !== undefined) + + const columnSet = new dbInstance.helpers.ColumnSet(columns, { + table: { + table: 'members', + }, + }) + if (Array.isArray(dbData.contributions)) { dbData.contributions = JSON.stringify(dbData.contributions) } diff --git a/services/libs/data-access-layer/src/members/identities.ts b/services/libs/data-access-layer/src/members/identities.ts index 10a2280837..0e1b05e33f 100644 --- a/services/libs/data-access-layer/src/members/identities.ts +++ b/services/libs/data-access-layer/src/members/identities.ts @@ -1,8 +1,9 @@ -import { DEFAULT_TENANT_ID } from '@crowd/common' +import { DEFAULT_TENANT_ID, generateUUIDv1 } from '@crowd/common' import { IMemberIdentity, + MemberIdentityDbInsert, + MemberIdentityDbRow, MemberIdentityType, - NewMemberIdentity, PlatformType, UpdateMemberIdentity, } from '@crowd/types' @@ -233,27 +234,32 @@ export async function findMemberIdByVerifiedIdentity( return result?.memberId ?? null } -export async function insertManyMemberIdentities( +export async function insertMemberIdentities( qx: QueryExecutor, - identities: NewMemberIdentity[], + identities: MemberIdentityDbInsert[], failOnConflict: boolean, returnRows: true, -): Promise -export async function insertManyMemberIdentities( +): Promise +export async function insertMemberIdentities( qx: QueryExecutor, - identities: NewMemberIdentity[], + identities: MemberIdentityDbInsert[], failOnConflict?: boolean, returnRows?: false, ): Promise -export async function insertManyMemberIdentities( +export async function insertMemberIdentities( qx: QueryExecutor, - identities: NewMemberIdentity[], + identities: MemberIdentityDbInsert[], failOnConflict = false, returnRows = false, -): Promise { +): Promise { + if (identities.length === 0) { + return returnRows ? [] : 0 + } + const query = prepareBulkInsert( 'memberIdentities', [ + 'id', 'memberId', 'tenantId', 'integrationId', @@ -265,12 +271,11 @@ export async function insertManyMemberIdentities( 'verified', 'verifiedBy', ], - identities.map((i) => { - return { - tenantId: DEFAULT_TENANT_ID, - ...i, - } - }), + identities.map((i) => ({ + ...i, + id: i.id ?? generateUUIDv1(), + tenantId: DEFAULT_TENANT_ID, + })), failOnConflict ? undefined : 'DO NOTHING', returnRows, ) @@ -282,32 +287,6 @@ export async function insertManyMemberIdentities( return qx.result(query) } -export async function createMemberIdentity( - qx: QueryExecutor, - i: NewMemberIdentity, - failOnConflict: boolean, - returnRows: true, -): Promise -export async function createMemberIdentity( - qx: QueryExecutor, - i: NewMemberIdentity, - failOnConflict?: boolean, - returnRows?: false, -): Promise -export async function createMemberIdentity( - qx: QueryExecutor, - i: NewMemberIdentity, - failOnConflict = false, - returnRows = false, -): Promise { - if (returnRows) { - const rows = await insertManyMemberIdentities(qx, [i], failOnConflict, true) - return rows[0] - } - - await insertManyMemberIdentities(qx, [i], failOnConflict) -} - export async function moveToNewMember( qx: QueryExecutor, p: { diff --git a/services/libs/data-access-layer/src/members/organizations.ts b/services/libs/data-access-layer/src/members/organizations.ts index 594ed21654..3a31a2a9d0 100644 --- a/services/libs/data-access-layer/src/members/organizations.ts +++ b/services/libs/data-access-layer/src/members/organizations.ts @@ -1,7 +1,10 @@ +import { generateUUIDv1 } from '@crowd/common' import { IMemberOrganization, IMemberOrganizationAffiliationOverride, IMemberRoleWithOrganization, + MemberOrganizationDbInsert, + MemberOrganizationDbRow, MemberOrganizationUpdate, OrganizationSource, } from '@crowd/types' @@ -14,6 +17,7 @@ import { import { deleteMemberSegmentAffiliations } from '../member_segment_affiliations' import { EntityType } from '../old/apps/script_executor_worker/types' import { QueryExecutor } from '../queryExecutor' +import { prepareBulkInsert } from '../utils' import { EmailDomainMemberOrganizationActivityDate } from './types' @@ -293,53 +297,89 @@ export async function fetchManyOrganizationAffiliationPolicies( ) } +export async function insertMemberOrganizations( + qx: QueryExecutor, + organizations: MemberOrganizationDbInsert[], + failOnConflict: boolean, + returnRows: true, +): Promise +export async function insertMemberOrganizations( + qx: QueryExecutor, + organizations: MemberOrganizationDbInsert[], + failOnConflict?: boolean, + returnRows?: false, +): Promise +export async function insertMemberOrganizations( + qx: QueryExecutor, + organizations: MemberOrganizationDbInsert[], + failOnConflict = false, + returnRows = false, +): Promise { + const ts = new Date() + + if (organizations.length === 0) { + return returnRows ? [] : 0 + } + + const query = prepareBulkInsert( + 'memberOrganizations', + [ + 'id', + 'memberId', + 'organizationId', + 'dateStart', + 'dateEnd', + 'title', + 'source', + 'verified', + 'verifiedBy', + 'createdAt', + 'updatedAt', + ], + organizations.map((o) => ({ + ...o, + id: o.id ?? generateUUIDv1(), + // NOT NULL, no DB default + createdAt: ts, + updatedAt: ts, + // NOT NULL DEFAULT false — must set while column is in INSERT list + verified: o.verified ?? false, + })), + failOnConflict ? undefined : 'DO NOTHING', + returnRows, + ) + + if (returnRows) { + return qx.select(query) + } + + return qx.result(query) +} + export async function createMemberOrganization( qx: QueryExecutor, memberId: string, data: Partial, ): Promise { - const result = await qx.selectOneOrNone( - ` - INSERT INTO "memberOrganizations"( - "memberId", - "organizationId", - "dateStart", - "dateEnd", - "title", - "source", - "verified", - "verifiedBy", - "createdAt", - "updatedAt" - ) - VALUES( - $(memberId), - $(organizationId), - $(dateStart), - $(dateEnd), - $(title), - $(source), - $(verified), - $(verifiedBy), - now(), - now() - ) - ON CONFLICT DO NOTHING - RETURNING id - `, - { - memberId, - organizationId: data.organizationId, - dateStart: data.dateStart ?? null, - dateEnd: data.dateEnd ?? null, - title: data.title ?? null, - source: data.source ?? null, - verified: data.verified ?? false, - verifiedBy: data.verifiedBy ?? null, - }, + const rows = await insertMemberOrganizations( + qx, + [ + { + memberId, + organizationId: data.organizationId, + dateStart: data.dateStart != null ? toIsoString(data.dateStart) : null, + dateEnd: data.dateEnd != null ? toIsoString(data.dateEnd) : null, + title: data.title, + source: data.source, + verified: data.verified, + verifiedBy: data.verifiedBy, + }, + ], + false, + true, ) - return result?.id + return rows[0]?.id } export async function createOrUpdateMemberOrganizations( diff --git a/services/libs/data-access-layer/src/members/projectAffiliations.ts b/services/libs/data-access-layer/src/members/projectAffiliations.ts index 92a3ebb08a..92e629001c 100644 --- a/services/libs/data-access-layer/src/members/projectAffiliations.ts +++ b/services/libs/data-access-layer/src/members/projectAffiliations.ts @@ -121,43 +121,6 @@ export async function fetchMemberSegmentAffiliationsForProject( ) } -export interface ISegmentAffiliationInsert { - organizationId: string - dateStart: string | null - dateEnd: string | null - verifiedBy: string -} - -/** - * Insert multiple segment affiliations for a member + project (segment) combination. - * All inserted affiliations are marked as verified. - */ -export async function insertMemberSegmentAffiliations( - qx: QueryExecutor, - memberId: string, - segmentId: string, - affiliations: ISegmentAffiliationInsert[], -): Promise { - for (const aff of affiliations) { - await qx.result( - ` - INSERT INTO "memberSegmentAffiliations" - (id, "memberId", "segmentId", "organizationId", "dateStart", "dateEnd", verified, "verifiedBy") - VALUES - (gen_random_uuid(), $(memberId), $(segmentId), $(organizationId), $(dateStart), $(dateEnd), true, $(verifiedBy)) - `, - { - memberId, - segmentId, - organizationId: aff.organizationId, - dateStart: aff.dateStart, - dateEnd: aff.dateEnd, - verifiedBy: aff.verifiedBy, - }, - ) - } -} - /** * Fetch work experiences for a member with organization details. * Used as fallback affiliations when no segment affiliations exist for a project. diff --git a/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/member.repo.ts b/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/member.repo.ts index 72e2ce285d..ae2bc305b5 100644 --- a/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/member.repo.ts +++ b/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/member.repo.ts @@ -3,10 +3,7 @@ import { DbColumnSet, DbStore, RepositoryBase } from '@crowd/database' import { Logger } from '@crowd/logging' import { IMemberIdentity, MemberIdentityType } from '@crowd/types' -import { - deleteManyMemberIdentities, - insertManyMemberIdentities, -} from '../../../../members/identities' +import { deleteManyMemberIdentities, insertMemberIdentities } from '../../../../members/identities' import { PgPromiseQueryExecutor } from '../../../../queryExecutor' import { IDbMember, getInsertMemberColumnSet, getSelectMemberColumnSet } from './member.data' @@ -101,11 +98,7 @@ export default class MemberRepository extends RepositoryBase { } }) - return insertManyMemberIdentities( - new PgPromiseQueryExecutor(this.db()), - objects, - failOnConflict, - ) + return insertMemberIdentities(new PgPromiseQueryExecutor(this.db()), objects, failOnConflict) } public async addToSegments(memberId: string, segmentIds: string[]): Promise { diff --git a/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/settings.repo.ts b/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/settings.repo.ts index 0c37a4e96e..ab60083eaa 100644 --- a/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/settings.repo.ts +++ b/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/settings.repo.ts @@ -1,6 +1,6 @@ import { DEFAULT_TENANT_ID } from '@crowd/common' import { DbStore, RepositoryBase } from '@crowd/database' -import { DEFAULT_ACTIVITY_TYPE_SETTINGS } from '@crowd/integrations' +import { DEFAULT_ACTIVITY_TYPE_SETTINGS } from '@crowd/integrations/src/integrations/activityTypes' import { Logger } from '@crowd/logging' import { PlatformType } from '@crowd/types' diff --git a/services/libs/data-access-layer/src/organizations/base.ts b/services/libs/data-access-layer/src/organizations/base.ts index 3323ba09fc..3d236d6407 100644 --- a/services/libs/data-access-layer/src/organizations/base.ts +++ b/services/libs/data-access-layer/src/organizations/base.ts @@ -12,16 +12,18 @@ import { IOrganizationIdentity, IQueryTimeseriesParams, ITimeseriesDatapoint, + OrganizationDbInsert, + OrganizationDbRow, OrganizationIdentityType, } from '@crowd/types' import { QueryExecutor } from '../queryExecutor' import { findLfSegmentByName } from '../segments' -import { QueryOptions, QueryResult, queryTable, queryTableById } from '../utils' +import { QueryOptions, QueryResult, prepareBulkInsert, queryTable, queryTableById } from '../utils' import { prepareSelectColumns } from '../utils' import { findOrgAttributes, markOrgAttributeDefault, upsertOrgAttributes } from './attributes' -import { addOrgIdentity, upsertOrgIdentities } from './identities' +import { insertOrganizationIdentities, upsertOrgIdentities } from './identities' import { IDbOrganization, IDbOrganizationInput } from './types' import { prepareOrganizationData } from './utils' @@ -363,6 +365,80 @@ export async function insertOrganization( return id } +export async function insertOrganizations( + qx: QueryExecutor, + organizations: OrganizationDbInsert[], + failOnConflict: boolean, + returnRows: true, +): Promise +export async function insertOrganizations( + qx: QueryExecutor, + organizations: OrganizationDbInsert[], + failOnConflict?: boolean, + returnRows?: false, +): Promise +export async function insertOrganizations( + qx: QueryExecutor, + organizations: OrganizationDbInsert[], + failOnConflict = false, + returnRows = false, +): Promise { + if (organizations.length === 0) { + return returnRows ? [] : 0 + } + + const ts = new Date() + + const query = prepareBulkInsert( + 'organizations', + [ + 'id', + 'displayName', + 'description', + 'logo', + 'tags', + 'employees', + 'revenueRange', + 'importHash', + 'location', + 'isTeamOrganization', + 'isAffiliationBlocked', + 'type', + 'size', + 'headline', + 'industry', + 'founded', + 'employeeChurnRate', + 'employeeGrowthRate', + 'manuallyCreated', + 'createdById', + 'updatedById', + 'tenantId', + 'createdAt', + 'updatedAt', + ], + organizations.map((o) => ({ + ...o, + id: o.id ?? generateUUIDv1(), + tenantId: DEFAULT_TENANT_ID, + createdAt: ts, + updatedAt: ts, + // NOT NULL DEFAULT false — must set while column is in INSERT list + isTeamOrganization: o.isTeamOrganization ?? false, + isAffiliationBlocked: o.isAffiliationBlocked ?? false, + manuallyCreated: o.manuallyCreated ?? false, + })), + failOnConflict ? undefined : 'DO NOTHING', + returnRows, + ) + + if (returnRows) { + return qx.select(query) + } + + return qx.result(query) +} + export async function updateOrganization( qe: QueryExecutor, organizationId: string, @@ -628,22 +704,25 @@ export async function findOrCreateOrganization( } // create identities - for (const i of data.identities) { - // add the identity + if (data.identities.length > 0) { await logExecutionTimeV2( - async () => - addOrgIdentity(qe, { - organizationId: id, - platform: i.platform, - type: i.type, - value: i.value, - verified: i.verified, - sourceId: i.sourceId, - integrationId, - source: i.source, - }), + () => + insertOrganizationIdentities( + qe, + data.identities.map((i) => ({ + organizationId: id, + platform: i.platform, + type: i.type, + value: i.value, + verified: i.verified, + sourceId: i.sourceId, + integrationId, + source: i.source, + })), + false, + ), log, - 'organizationService -> findOrCreateOrganization -> addOrgIdentity', + 'organizationService -> findOrCreateOrganization -> insertOrganizationIdentities', ) } } diff --git a/services/libs/data-access-layer/src/organizations/identities.ts b/services/libs/data-access-layer/src/organizations/identities.ts index 14aa5956a3..456318fb74 100644 --- a/services/libs/data-access-layer/src/organizations/identities.ts +++ b/services/libs/data-access-layer/src/organizations/identities.ts @@ -1,12 +1,13 @@ import { DEFAULT_TENANT_ID } from '@crowd/common' import { IOrganizationIdentity, - NewOrganizationIdentity, + type OrganizationIdentityDbInsert, + type OrganizationIdentityDbRow, OrganizationIdentityType, } from '@crowd/types' import { QueryExecutor } from '../queryExecutor' -import { QueryOptions, QueryResult, queryTable } from '../utils' +import { QueryOptions, QueryResult, prepareBulkInsert, queryTable } from '../utils' import { IDbOrgIdentityUpdateInput } from './types' @@ -71,26 +72,54 @@ export async function updateOrgIdentityVerifiedFlag( ) } -export async function addOrgIdentity(qx: QueryExecutor, identity: NewOrganizationIdentity) { - return qx.result( - ` - INSERT INTO "organizationIdentities" ( - "organizationId", - "source", - platform, - value, - type, - verified, - "sourceId", - "tenantId", - "integrationId", - "createdAt" - ) - VALUES ($(organizationId), $(source), $(platform), $(value), $(type), $(verified), $(sourceId), $(tenantId), $(integrationId), NOW()) - ON CONFLICT DO NOTHING; - `, - { tenantId: DEFAULT_TENANT_ID, ...identity }, +export async function insertOrganizationIdentities( + qx: QueryExecutor, + identities: OrganizationIdentityDbInsert[], + failOnConflict: boolean, + returnRows: true, +): Promise +export async function insertOrganizationIdentities( + qx: QueryExecutor, + identities: OrganizationIdentityDbInsert[], + failOnConflict?: boolean, + returnRows?: false, +): Promise +export async function insertOrganizationIdentities( + qx: QueryExecutor, + identities: OrganizationIdentityDbInsert[], + failOnConflict = false, + returnRows = false, +): Promise { + if (identities.length === 0) { + return returnRows ? [] : 0 + } + + const query = prepareBulkInsert( + 'organizationIdentities', + [ + 'organizationId', + 'platform', + 'value', + 'type', + 'verified', + 'source', + 'sourceId', + 'tenantId', + 'integrationId', + ], + identities.map((i) => ({ + ...i, + tenantId: DEFAULT_TENANT_ID, + })), + failOnConflict ? undefined : 'DO NOTHING', + returnRows, ) + + if (returnRows) { + return qx.select(query) + } + + return qx.result(query) } export async function upsertOrgIdentities( @@ -100,47 +129,42 @@ export async function upsertOrgIdentities( integrationId?: string, ): Promise { const existingIdentities = await fetchOrgIdentities(qe, organizationId) - const toCreate = [] - const toUpdate = [] + const toCreate: OrganizationIdentityDbInsert[] = [] + const toUpdate: Partial[] = [] for (const i of identities) { const existing = existingIdentities.find( (ei) => ei.value === i.value && ei.platform === i.platform && ei.type === i.type, ) if (!existing) { - toCreate.push(i) - } else if (existing && existing.verified !== i.verified) { - toUpdate.push(i) - } - } - - if (toCreate.length > 0) { - for (const i of toCreate) { - // add the identity - await addOrgIdentity(qe, { + toCreate.push({ organizationId, platform: i.platform, type: i.type, value: i.value, - verified: i.verified, - source: i.source, - sourceId: i.sourceId, - integrationId, + // NOT NULL DEFAULT false — undefined would insert NULL + verified: i.verified ?? false, + source: i.source ?? null, + sourceId: i.sourceId ?? null, + integrationId: integrationId ?? null, }) + } else if (i.verified !== undefined && existing.verified !== i.verified) { + toUpdate.push(i) } } - if (toUpdate.length > 0) { - for (const i of toUpdate) { - // update the identity - await updateOrgIdentityVerifiedFlag(qe, { - organizationId, - platform: i.platform, - type: i.type, - value: i.value, - verified: i.verified, - }) - } + if (toCreate.length > 0) { + await insertOrganizationIdentities(qe, toCreate, false) + } + + for (const i of toUpdate) { + await updateOrgIdentityVerifiedFlag(qe, { + organizationId, + platform: i.platform, + type: i.type, + value: i.value, + verified: i.verified, + }) } } diff --git a/services/libs/data-access-layer/src/segments/index.ts b/services/libs/data-access-layer/src/segments/index.ts index 19498a39a5..7fbd056a3d 100644 --- a/services/libs/data-access-layer/src/segments/index.ts +++ b/services/libs/data-access-layer/src/segments/index.ts @@ -1,18 +1,22 @@ import lodash from 'lodash' import cloneDeep from 'lodash.clonedeep' -import { DEFAULT_TENANT_ID } from '@crowd/common' -import { DEFAULT_ACTIVITY_TYPE_SETTINGS } from '@crowd/integrations' +import { DEFAULT_TENANT_ID, generateUUIDv1 } from '@crowd/common' +import { DEFAULT_ACTIVITY_TYPE_SETTINGS } from '@crowd/integrations/src/integrations/activityTypes' import { ActivityTypeSettings, MergeActionState, MergeActionType, PlatformType, SegmentData, + SegmentDbInsert, + SegmentDbRow, SegmentRawData, + SegmentStatus, } from '@crowd/types' import { QueryExecutor } from '../queryExecutor' +import { prepareBulkInsert } from '../utils' export async function findProjectGroupByName( qx: QueryExecutor, @@ -164,6 +168,71 @@ export function isSegmentSubproject(segment: SegmentData | SegmentRawData): bool return segment.slug != null && segment.parentSlug != null && segment.grandparentSlug != null } +export async function insertSegments( + qx: QueryExecutor, + segments: SegmentDbInsert[], + failOnConflict: boolean, + returnRows: true, +): Promise +export async function insertSegments( + qx: QueryExecutor, + segments: SegmentDbInsert[], + failOnConflict?: boolean, + returnRows?: false, +): Promise +export async function insertSegments( + qx: QueryExecutor, + segments: SegmentDbInsert[], + failOnConflict = false, + returnRows = false, +): Promise { + if (segments.length === 0) { + return returnRows ? [] : 0 + } + + const query = prepareBulkInsert( + 'segments', + [ + 'id', + 'slug', + 'name', + 'url', + 'status', + 'isLF', + 'parentName', + 'grandparentName', + 'parentSlug', + 'grandparentSlug', + 'description', + 'sourceId', + 'sourceParentId', + 'customActivityTypes', + 'activityChannels', + 'parentId', + 'grandparentId', + // `type` is GENERATED ALWAYS from parentSlug/grandparentSlug — do not insert + 'maturity', + 'tenantId', + ], + segments.map((s) => ({ + ...s, + id: s.id ?? generateUUIDv1(), + tenantId: DEFAULT_TENANT_ID, + // NOT NULL with defaults — must set while column is in INSERT list + status: s.status ?? SegmentStatus.ACTIVE, + isLF: s.isLF ?? true, + })), + failOnConflict ? undefined : 'DO NOTHING', + returnRows, + ) + + if (returnRows) { + return qx.select(query) + } + + return qx.result(query) +} + export async function getSegmentSubprojects( qx: QueryExecutor, segmentIds: string[], diff --git a/services/libs/test-kit/package.json b/services/libs/test-kit/package.json index 2ba5fe2ebf..365ca89364 100644 --- a/services/libs/test-kit/package.json +++ b/services/libs/test-kit/package.json @@ -1,8 +1,11 @@ { "name": "@crowd/test-kit", "private": true, + "sideEffects": false, "exports": { - "./fixtures": "./src/fixtures.ts" + "./db": "./src/db/index.ts", + "./factories": "./src/factories/index.ts", + "./faker": "./src/faker.ts" }, "scripts": { "lint": "npx eslint --ext .ts src --max-warnings=0", @@ -12,7 +15,11 @@ }, "dependencies": { "@crowd/common": "workspace:*", + "@crowd/data-access-layer": "workspace:*", "@crowd/database": "workspace:*", + "@crowd/integrations": "workspace:*", + "@crowd/types": "workspace:*", + "@faker-js/faker": "^9.9.0", "pg-promise": "^11.4.3" }, "devDependencies": { diff --git a/services/libs/test-kit/src/db/index.ts b/services/libs/test-kit/src/db/index.ts new file mode 100644 index 0000000000..1c0dbaaf69 --- /dev/null +++ b/services/libs/test-kit/src/db/index.ts @@ -0,0 +1,2 @@ +export { openTestWorkerDatabase, resetTestDatabase, seedTestBaseline } from './postgres' +export { withQx } from './qx' diff --git a/services/libs/test-kit/src/postgres.ts b/services/libs/test-kit/src/db/postgres.ts similarity index 98% rename from services/libs/test-kit/src/postgres.ts rename to services/libs/test-kit/src/db/postgres.ts index c69878b86d..7303ce5434 100644 --- a/services/libs/test-kit/src/postgres.ts +++ b/services/libs/test-kit/src/db/postgres.ts @@ -4,7 +4,12 @@ import { DEFAULT_TENANT_ID, IS_TEST_ENV } from '@crowd/common' import { type DbConnection, type DbInstance, type IDatabaseConfig } from '@crowd/database' import type { QueryExecutor } from '@crowd/database' -import type { TestPostgres } from './types' +type TestPostgres = { + host: string + port: number + user: string + password: string +} /** * Creates an isolated, dynamic test database cloned from the template database diff --git a/services/libs/test-kit/src/fixtures.ts b/services/libs/test-kit/src/db/qx.ts similarity index 100% rename from services/libs/test-kit/src/fixtures.ts rename to services/libs/test-kit/src/db/qx.ts diff --git a/services/libs/test-kit/src/factories/activity-relation.ts b/services/libs/test-kit/src/factories/activity-relation.ts new file mode 100644 index 0000000000..73c1edea65 --- /dev/null +++ b/services/libs/test-kit/src/factories/activity-relation.ts @@ -0,0 +1,169 @@ +import { faker } from '@faker-js/faker' + +import { generateUUIDv1 } from '@crowd/common' +import { createOrUpdateRelations } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import { DEFAULT_ACTIVITY_TYPE_SETTINGS } from '@crowd/integrations/src/integrations/activityTypes' +import { + type ActivityRelationDbInsert, + type MemberIdentityDbRow, + type MemberOrganizationDbRow, + PlatformType, +} from '@crowd/types' + +import { withDefaults } from './defaults' + +const DEFAULT_COUNT = 100 + +interface GenerateActivityRelationsInput { + memberId: string + segmentId: string + identities: MemberIdentityDbRow[] + timestamp: { + from: string + to: string + } + count?: number + memberOrganizations?: MemberOrganizationDbRow[] +} + +export const withActivityRelationDefaults = withDefaults()({ + activityId: () => generateUUIDv1(), + sourceId: () => generateUUIDv1(), +}) + +/** Build N activityRelation rows from member context. */ +export function generateActivityRelations( + input: GenerateActivityRelationsInput, +): ActivityRelationDbInsert[] { + const { + memberId, + segmentId, + identities, + timestamp, + memberOrganizations, + count = DEFAULT_COUNT, + } = input + + if (identities.length === 0) { + throw new Error('generateActivityRelations requires at least one identity') + } + + if (count < 1) { + throw new Error('generateActivityRelations count must be >= 1') + } + + const fromMs = new Date(timestamp.from).getTime() + const toMs = new Date(timestamp.to).getTime() + + if (Number.isNaN(fromMs) || Number.isNaN(toMs)) { + throw new Error('generateActivityRelations timestamp.from/to must be valid dates') + } + + if (fromMs > toMs) { + throw new Error('generateActivityRelations timestamp.from must be <= timestamp.to') + } + + // Same channel for all rows of a given platform. + const channelByPlatform = new Map() + + return Array.from({ length: count }, (_, index) => { + const identity = identities[index % identities.length] + const platform = identity.platform as PlatformType + + // Evenly spaced across [from, to]. + const activityTimestamp = + count === 1 ? new Date(fromMs) : new Date(fromMs + ((toMs - fromMs) * index) / (count - 1)) + + let channel = channelByPlatform.get(platform) + if (!channel) { + channel = channelFor(platform) + channelByPlatform.set(platform, channel) + } + + return { + activityId: generateUUIDv1(), + memberId, + segmentId, + platform, + username: identity.value, + sourceId: generateUUIDv1(), + type: faker.helpers.arrayElement(activityTypesFor(platform)), + timestamp: activityTimestamp.toISOString(), + channel, + organizationId: resolveOrganizationId(activityTimestamp, memberOrganizations), + score: faker.number.int({ min: 0, max: 10 }), + } + }) +} + +/** Insert activityRelation rows. */ +export async function createActivityRelations( + qx: QueryExecutor, + data: ActivityRelationDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + await createOrUpdateRelations(qx, data as Parameters[1], true) + + return data +} + +function activityTypesFor(platform: PlatformType): string[] { + // Treat github-nango as github for activity types. + const key = platform === PlatformType.GITHUB_NANGO ? PlatformType.GITHUB : platform + + const types = Object.keys(DEFAULT_ACTIVITY_TYPE_SETTINGS[key] ?? {}) + + return types.length > 0 ? types : ['activity'] +} + +function channelFor(platform: PlatformType): string { + const org = faker.internet.domainWord() + const repo = faker.internet.domainWord() + + switch (platform) { + case PlatformType.GITHUB: + case PlatformType.GITHUB_NANGO: + case PlatformType.GIT: + return `https://github.com/${org}/${repo}` + + case PlatformType.GITLAB: + return `https://gitlab.com/${org}/${repo}` + + case PlatformType.DISCORD: + case PlatformType.SLACK: + return `#${faker.word.noun()}` + + default: + return `${org}/${repo}` + } +} + +/** Pick organizationId for a timestamp from the given stints. */ +function resolveOrganizationId(timestamp: Date, stints?: MemberOrganizationDbRow[]): string | null { + if (!stints?.length) { + return null + } + + let fallback: string | null = null + + for (const stint of stints) { + // No dates → keep as fallback; dated match returns immediately. + if (!stint.dateStart && !stint.dateEnd) { + fallback ??= stint.organizationId + continue + } + + const start = stint.dateStart ? new Date(stint.dateStart) : undefined + const end = stint.dateEnd ? new Date(stint.dateEnd) : undefined + + if ((!start || timestamp >= start) && (!end || timestamp <= end)) { + return stint.organizationId + } + } + + return fallback +} diff --git a/services/libs/test-kit/src/factories/defaults.ts b/services/libs/test-kit/src/factories/defaults.ts new file mode 100644 index 0000000000..d0938b84eb --- /dev/null +++ b/services/libs/test-kit/src/factories/defaults.ts @@ -0,0 +1,44 @@ +type DefaultValue = V | (() => V) + +type DefaultsFor = { + [P in K]: DefaultValue +} + +/** Required keys of T stay required; keys covered by defaults become optional. */ +type WithDefaultsRow = Omit & Partial> + +function resolveDefault(value: DefaultValue): V { + if (typeof value === 'function') { + return (value as () => V)() + } + return value as V +} + +function applyDefaults( + row: WithDefaultsRow, + defaults: DefaultsFor, +): T { + const result = { ...row } as T + + for (const key of Object.keys(defaults) as K[]) { + if (result[key] === undefined) { + result[key] = resolveDefault(defaults[key]) as T[K] + } + } + + return result +} + +/** + * Fills missing allowlisted fields on each row. Explicit values always win. + * Keys provided in `defaults` become optional on input; all other required + * keys of T remain required at compile time. + * + * Usage: `withDefaults()({ id: () => ..., displayName: () => ... })` + */ +export function withDefaults() { + return (defaults: DefaultsFor) => { + return (rows: Array>): T[] => + rows.map((row) => applyDefaults(row, defaults)) + } +} diff --git a/services/libs/test-kit/src/factories/index.ts b/services/libs/test-kit/src/factories/index.ts new file mode 100644 index 0000000000..0038d84cd0 --- /dev/null +++ b/services/libs/test-kit/src/factories/index.ts @@ -0,0 +1,10 @@ +export * from './defaults' +export * from './member' +export * from './member-identity' +export * from './organization' +export * from './organization-identity' +export * from './member-organization' +export * from './member-organization-affiliation-override' +export * from './segment' +export * from './member-segment-affiliation' +export * from './activity-relation' diff --git a/services/libs/test-kit/src/factories/member-identity.ts b/services/libs/test-kit/src/factories/member-identity.ts new file mode 100644 index 0000000000..e5a88df7a0 --- /dev/null +++ b/services/libs/test-kit/src/factories/member-identity.ts @@ -0,0 +1,21 @@ +import { generateUUIDv1 } from '@crowd/common' +import { insertMemberIdentities } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { MemberIdentityDbInsert, MemberIdentityDbRow } from '@crowd/types' + +import { withDefaults } from './defaults' + +export const withMemberIdentityDefaults = withDefaults()({ + id: () => generateUUIDv1(), +}) + +export async function createMemberIdentities( + qx: QueryExecutor, + data: MemberIdentityDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return insertMemberIdentities(qx, data, true, true) +} diff --git a/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts b/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts new file mode 100644 index 0000000000..d1108a7647 --- /dev/null +++ b/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts @@ -0,0 +1,17 @@ +import { changeMemberOrganizationAffiliationOverrides } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { + MemberOrganizationAffiliationOverrideDbInsert, + MemberOrganizationAffiliationOverrideDbRow, +} from '@crowd/types' + +export async function upsertMemberOrganizationAffiliationOverrides( + qx: QueryExecutor, + data: MemberOrganizationAffiliationOverrideDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return changeMemberOrganizationAffiliationOverrides(qx, data, true) +} diff --git a/services/libs/test-kit/src/factories/member-organization.ts b/services/libs/test-kit/src/factories/member-organization.ts new file mode 100644 index 0000000000..8b3f6b4c71 --- /dev/null +++ b/services/libs/test-kit/src/factories/member-organization.ts @@ -0,0 +1,14 @@ +import { insertMemberOrganizations } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { MemberOrganizationDbInsert, MemberOrganizationDbRow } from '@crowd/types' + +export async function createMemberOrganizations( + qx: QueryExecutor, + data: MemberOrganizationDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return insertMemberOrganizations(qx, data, true, true) +} diff --git a/services/libs/test-kit/src/factories/member-segment-affiliation.ts b/services/libs/test-kit/src/factories/member-segment-affiliation.ts new file mode 100644 index 0000000000..01a11ee73a --- /dev/null +++ b/services/libs/test-kit/src/factories/member-segment-affiliation.ts @@ -0,0 +1,14 @@ +import { insertMemberSegmentAffiliations } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { MemberSegmentAffiliationDbInsert, MemberSegmentAffiliationDbRow } from '@crowd/types' + +export async function createMemberSegmentAffiliations( + qx: QueryExecutor, + data: MemberSegmentAffiliationDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return insertMemberSegmentAffiliations(qx, data, true, true) +} diff --git a/services/libs/test-kit/src/factories/member.ts b/services/libs/test-kit/src/factories/member.ts index e69de29bb2..8bdbcaf57d 100644 --- a/services/libs/test-kit/src/factories/member.ts +++ b/services/libs/test-kit/src/factories/member.ts @@ -0,0 +1,26 @@ +import { faker } from '@faker-js/faker' + +import { generateUUIDv1 } from '@crowd/common' +import { createMember as insertMember } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { MemberDbInsert, MemberDbRow } from '@crowd/types' + +import { withDefaults } from './defaults' + +export const withMemberDefaults = withDefaults()({ + id: () => generateUUIDv1(), + displayName: () => faker.person.fullName(), + joinedAt: () => new Date().toISOString(), + manuallyCreated: false, +}) + +export async function createMembers( + qx: QueryExecutor, + data: MemberDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return Promise.all(data.map((item) => insertMember(qx, item))) +} diff --git a/services/libs/test-kit/src/factories/organization-identity.ts b/services/libs/test-kit/src/factories/organization-identity.ts new file mode 100644 index 0000000000..9a724cf05b --- /dev/null +++ b/services/libs/test-kit/src/factories/organization-identity.ts @@ -0,0 +1,15 @@ +import { insertOrganizationIdentities } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { OrganizationIdentityDbInsert, OrganizationIdentityDbRow } from '@crowd/types' + +/** Persist organization identity rows via the production insert path. */ +export async function createOrganizationIdentities( + qx: QueryExecutor, + data: OrganizationIdentityDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return insertOrganizationIdentities(qx, data, true, true) +} diff --git a/services/libs/test-kit/src/factories/organization.ts b/services/libs/test-kit/src/factories/organization.ts new file mode 100644 index 0000000000..7e65b16961 --- /dev/null +++ b/services/libs/test-kit/src/factories/organization.ts @@ -0,0 +1,27 @@ +import { faker } from '@faker-js/faker' + +import { generateUUIDv1 } from '@crowd/common' +import { insertOrganizations } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { OrganizationDbInsert, OrganizationDbRow } from '@crowd/types' + +import { withDefaults } from './defaults' + +export const withOrganizationDefaults = withDefaults()({ + id: () => generateUUIDv1(), + displayName: () => faker.company.name(), + manuallyCreated: false, + isTeamOrganization: false, + isAffiliationBlocked: false, +}) + +export async function createOrganizations( + qx: QueryExecutor, + data: OrganizationDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return insertOrganizations(qx, data, true, true) +} diff --git a/services/libs/test-kit/src/factories/segment.ts b/services/libs/test-kit/src/factories/segment.ts new file mode 100644 index 0000000000..c4b2eed26f --- /dev/null +++ b/services/libs/test-kit/src/factories/segment.ts @@ -0,0 +1,178 @@ +import { generateUUIDv1 } from '@crowd/common' +import { insertSegments } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import { type SegmentDbInsert, type SegmentDbRow, SegmentStatus } from '@crowd/types' + +import { withDefaults } from './defaults' + +/** Hierarchy fields are derived by createSegments; callers must supply name + slug. */ +type SegmentNodeInput = { + name: string + slug: string +} & Partial< + Omit< + SegmentDbInsert, + | 'name' + | 'slug' + | 'parentId' + | 'grandparentId' + | 'parentSlug' + | 'grandparentSlug' + | 'parentName' + | 'grandparentName' + > +> + +export type CreateSubproject = SegmentNodeInput +export type CreateProject = SegmentNodeInput & { + subprojects?: CreateSubproject[] +} +export type CreateProjectGroup = SegmentNodeInput & { + projects?: CreateProject[] +} + +export type CreateSegmentsResult = { + projectGroups: Array<{ + row: SegmentDbRow + projects: Array<{ + row: SegmentDbRow + subprojects: SegmentDbRow[] + }> + }> +} + +const applySegmentNodeDefaults = withDefaults()({ + id: () => generateUUIDv1(), + status: SegmentStatus.ACTIVE, + isLF: true, +}) + +/** + * Fills missing allowlisted fields on each node. Does not invent children. + * Caller still owns name, slug, and tree shape. Explicit values always win. + */ +export function withSegmentDefaults(trees: CreateProjectGroup[]): CreateProjectGroup[] { + return trees.map((group) => { + const { projects, ...groupFields } = group + const [projectGroup] = applySegmentNodeDefaults([groupFields]) + + return { + ...projectGroup, + projects: projects?.map((project) => { + const { subprojects, ...projectFields } = project + const [projectDefaults] = applySegmentNodeDefaults([projectFields]) + + return { + ...projectDefaults, + subprojects: subprojects ? applySegmentNodeDefaults(subprojects) : subprojects, + } + }), + } + }) +} + +/** + * Inserts nested segment trees in one go. Derives parent/grandparent fields from + * structure (`type` is generated by the DB from parentSlug/grandparentSlug). + * Requires name + slug on every node. + */ +export async function createSegments( + qx: QueryExecutor, + trees: CreateProjectGroup[], +): Promise { + const rowsToInsert: SegmentDbInsert[] = [] + const structure: Array<{ + projectGroupId: string + projects: Array<{ projectId: string; subprojectIds: string[] }> + }> = [] + + for (const group of trees) { + const { projects: groupProjects = [], ...groupFields } = group + const projectGroupId = groupFields.id ?? generateUUIDv1() + const projectGroupName = groupFields.name + const projectGroupSlug = groupFields.slug + + rowsToInsert.push({ + ...groupFields, + id: projectGroupId, + name: projectGroupName, + slug: projectGroupSlug, + parentId: null, + parentSlug: null, + parentName: null, + grandparentId: null, + grandparentSlug: null, + grandparentName: null, + }) + + const projectStructs: Array<{ projectId: string; subprojectIds: string[] }> = [] + + for (const project of groupProjects) { + const { subprojects: projectSubprojects = [], ...projectFields } = project + const projectId = projectFields.id ?? generateUUIDv1() + const projectName = projectFields.name + const projectSlug = projectFields.slug + + rowsToInsert.push({ + ...projectFields, + id: projectId, + name: projectName, + slug: projectSlug, + parentId: projectGroupId, + parentSlug: projectGroupSlug, + parentName: projectGroupName, + grandparentId: null, + grandparentSlug: null, + grandparentName: null, + }) + + const subprojectIds: string[] = [] + + for (const subproject of projectSubprojects) { + const subprojectId = subproject.id ?? generateUUIDv1() + const subprojectName = subproject.name + const subprojectSlug = subproject.slug + + rowsToInsert.push({ + ...subproject, + id: subprojectId, + name: subprojectName, + slug: subprojectSlug, + parentId: projectId, + parentSlug: projectSlug, + parentName: projectName, + grandparentId: projectGroupId, + grandparentSlug: projectGroupSlug, + grandparentName: projectGroupName, + }) + + subprojectIds.push(subprojectId) + } + + projectStructs.push({ projectId, subprojectIds }) + } + + structure.push({ projectGroupId, projects: projectStructs }) + } + + const rows = await insertSegments(qx, rowsToInsert, true, true) + const byId = new Map(rows.map((row) => [row.id, row])) + + const requireRow = (id: string): SegmentDbRow => { + const row = byId.get(id) + if (!row) { + throw new Error(`createSegments: inserted segment ${id} was not returned`) + } + return row + } + + return { + projectGroups: structure.map((group) => ({ + row: requireRow(group.projectGroupId), + projects: group.projects.map((project) => ({ + row: requireRow(project.projectId), + subprojects: project.subprojectIds.map((id) => requireRow(id)), + })), + })), + } +} diff --git a/services/libs/test-kit/src/faker.ts b/services/libs/test-kit/src/faker.ts new file mode 100644 index 0000000000..0601a53453 --- /dev/null +++ b/services/libs/test-kit/src/faker.ts @@ -0,0 +1 @@ +export { faker } from '@faker-js/faker' diff --git a/services/libs/test-kit/src/types.ts b/services/libs/test-kit/src/types.ts deleted file mode 100644 index 902271f076..0000000000 --- a/services/libs/test-kit/src/types.ts +++ /dev/null @@ -1,6 +0,0 @@ -export type TestPostgres = { - host: string - port: number - user: string - password: string -} diff --git a/services/libs/types/src/db/activity-relations.ts b/services/libs/types/src/db/activity-relations.ts new file mode 100644 index 0000000000..68b199a825 --- /dev/null +++ b/services/libs/types/src/db/activity-relations.ts @@ -0,0 +1,55 @@ +export interface ActivityRelationDbRow { + activityId: string + memberId: string + objectMemberId: string | null + organizationId: string | null + conversationId: string | null + parentId: string | null + segmentId: string + platform: string + username: string + objectMemberUsername: string | null + createdAt: string + updatedAt: string + sourceId: string | null + sourceParentId: string | null + type: string | null + timestamp: string | null + channel: string | null + sentimentScore: number | null + gitInsertions: number | null + gitDeletions: number | null + score: number | null + isContribution: boolean | null + pullRequestReviewState: string | null +} + +export type ActivityRelationDbInsert = Pick< + ActivityRelationDbRow, + | 'activityId' + | 'memberId' + | 'segmentId' + | 'platform' + | 'username' + | 'sourceId' + | 'type' + | 'timestamp' + | 'channel' +> & + Partial< + Omit< + ActivityRelationDbRow, + | 'activityId' + | 'memberId' + | 'segmentId' + | 'platform' + | 'username' + | 'sourceId' + | 'type' + | 'timestamp' + | 'channel' + | 'createdAt' + | 'updatedAt' + | 'isContribution' + > + > diff --git a/services/libs/types/src/db/index.ts b/services/libs/types/src/db/index.ts new file mode 100644 index 0000000000..89237068ad --- /dev/null +++ b/services/libs/types/src/db/index.ts @@ -0,0 +1,9 @@ +export * from './activity-relations' +export * from './members' +export * from './member-identities' +export * from './member-organizations' +export * from './member-organization-affiliation-overrides' +export * from './member-segment-affiliations' +export * from './organizations' +export * from './organization-identities' +export * from './segments' diff --git a/services/libs/types/src/db/member-identities.ts b/services/libs/types/src/db/member-identities.ts new file mode 100644 index 0000000000..5590e4bad4 --- /dev/null +++ b/services/libs/types/src/db/member-identities.ts @@ -0,0 +1,37 @@ +import type { MemberIdentityType } from '../enums/members' + +export interface MemberIdentityDbRow { + id: string + memberId: string + tenantId: string + platform: string + value: string + type: MemberIdentityType + verified: boolean + verifiedBy: string | null + source: string | null + sourceId: string | null + integrationId: string | null + createdAt: string + updatedAt: string + deletedAt: string | null +} + +export type MemberIdentityDbInsert = Pick< + MemberIdentityDbRow, + 'memberId' | 'platform' | 'value' | 'type' | 'verified' +> & + Partial< + Omit< + MemberIdentityDbRow, + | 'memberId' + | 'platform' + | 'value' + | 'type' + | 'verified' + | 'tenantId' + | 'createdAt' + | 'updatedAt' + | 'deletedAt' + > + > diff --git a/services/libs/types/src/db/member-organization-affiliation-overrides.ts b/services/libs/types/src/db/member-organization-affiliation-overrides.ts new file mode 100644 index 0000000000..dec9385c2f --- /dev/null +++ b/services/libs/types/src/db/member-organization-affiliation-overrides.ts @@ -0,0 +1,20 @@ +export interface MemberOrganizationAffiliationOverrideDbRow { + id: string + memberId: string + memberOrganizationId: string + allowAffiliation: boolean | null + isPrimaryWorkExperience: boolean | null +} + +export type MemberOrganizationAffiliationOverrideDbInsert = Pick< + MemberOrganizationAffiliationOverrideDbRow, + 'memberId' | 'memberOrganizationId' +> & + Partial< + Pick & { + allowAffiliation: NonNullable + isPrimaryWorkExperience: NonNullable< + MemberOrganizationAffiliationOverrideDbRow['isPrimaryWorkExperience'] + > + } + > diff --git a/services/libs/types/src/db/member-organizations.ts b/services/libs/types/src/db/member-organizations.ts new file mode 100644 index 0000000000..494f6f269d --- /dev/null +++ b/services/libs/types/src/db/member-organizations.ts @@ -0,0 +1,25 @@ +export interface MemberOrganizationDbRow { + id: string + memberId: string + organizationId: string + dateStart: string | null + dateEnd: string | null + title: string | null + source: string | null + verified: boolean + verifiedBy: string | null + createdAt: string + updatedAt: string + deletedAt: string | null +} + +export type MemberOrganizationDbInsert = Pick< + MemberOrganizationDbRow, + 'memberId' | 'organizationId' +> & + Partial< + Omit< + MemberOrganizationDbRow, + 'memberId' | 'organizationId' | 'createdAt' | 'updatedAt' | 'deletedAt' + > + > diff --git a/services/libs/types/src/db/member-segment-affiliations.ts b/services/libs/types/src/db/member-segment-affiliations.ts new file mode 100644 index 0000000000..b1a3c1eb08 --- /dev/null +++ b/services/libs/types/src/db/member-segment-affiliations.ts @@ -0,0 +1,17 @@ +export interface MemberSegmentAffiliationDbRow { + id: string + memberId: string + segmentId: string + organizationId: string | null + dateStart: string | null + dateEnd: string | null + verified: boolean + verifiedBy: string | null + deletedAt: string | null +} + +export type MemberSegmentAffiliationDbInsert = Pick< + MemberSegmentAffiliationDbRow, + 'memberId' | 'segmentId' +> & + Partial> diff --git a/services/libs/types/src/db/members.ts b/services/libs/types/src/db/members.ts new file mode 100644 index 0000000000..6a7cd701b0 --- /dev/null +++ b/services/libs/types/src/db/members.ts @@ -0,0 +1,29 @@ +import type { IMemberContribution } from '../members' + +export interface MemberDbRow { + id: string + displayName: string + joinedAt: string + tenantId: string + createdAt: string + updatedAt: string + manuallyCreated: boolean + attributes: Record | null + reach: Record | null + score: number | null + contributions: IMemberContribution[] | null + enrichedBy: string[] | null + manuallyChangedFields: string[] | null + importHash: string | null + deletedAt: string | null + createdById: string | null + updatedById: string | null +} + +export type MemberDbInsert = Pick & + Partial< + Omit< + MemberDbRow, + 'displayName' | 'joinedAt' | 'tenantId' | 'createdAt' | 'updatedAt' | 'deletedAt' + > + > diff --git a/services/libs/types/src/db/organization-identities.ts b/services/libs/types/src/db/organization-identities.ts new file mode 100644 index 0000000000..0d41e9bcf7 --- /dev/null +++ b/services/libs/types/src/db/organization-identities.ts @@ -0,0 +1,21 @@ +import type { OrganizationIdentityType } from '../enums/organizations' + +export interface OrganizationIdentityDbRow { + organizationId: string + platform: string + value: string + type: OrganizationIdentityType + verified: boolean + source: string | null + sourceId: string | null + integrationId: string | null + tenantId: string + createdAt: string + updatedAt: string +} + +export type OrganizationIdentityDbInsert = Pick< + OrganizationIdentityDbRow, + 'organizationId' | 'platform' | 'value' | 'type' | 'verified' +> & + Partial> diff --git a/services/libs/types/src/db/organizations.ts b/services/libs/types/src/db/organizations.ts new file mode 100644 index 0000000000..4382fe9c2c --- /dev/null +++ b/services/libs/types/src/db/organizations.ts @@ -0,0 +1,32 @@ +export interface OrganizationDbRow { + id: string + displayName: string | null + description: string | null + logo: string | null + tags: string[] | null + employees: number | null + revenueRange: Record | null + importHash: string | null + location: string | null + isTeamOrganization: boolean + isAffiliationBlocked: boolean + type: string | null + size: string | null + headline: string | null + industry: string | null + founded: number | null + employeeChurnRate: Record | null + employeeGrowthRate: Record | null + manuallyCreated: boolean + lastEnrichedAt: string | null + tenantId: string + createdAt: string + updatedAt: string + deletedAt: string | null + createdById: string | null + updatedById: string | null +} + +export type OrganizationDbInsert = Partial< + Omit +> diff --git a/services/libs/types/src/db/segments.ts b/services/libs/types/src/db/segments.ts new file mode 100644 index 0000000000..788a126185 --- /dev/null +++ b/services/libs/types/src/db/segments.ts @@ -0,0 +1,35 @@ +import type { SegmentStatus, SegmentType } from '../segments' + +export interface SegmentDbRow { + id: string + slug: string + tenantId: string + status: SegmentStatus + isLF: boolean + createdAt: string + updatedAt: string + url: string | null + name: string | null + parentName: string | null + grandparentName: string | null + parentSlug: string | null + grandparentSlug: string | null + description: string | null + sourceId: string | null + sourceParentId: string | null + customActivityTypes: Record | null + activityChannels: Record | null + dashboardCacheLastRefreshedAt: string | null + parentId: string | null + grandparentId: string | null + type: SegmentType + maturity: string | null +} + +export type SegmentDbInsert = Pick & + Partial< + Omit< + SegmentDbRow, + 'slug' | 'tenantId' | 'createdAt' | 'updatedAt' | 'dashboardCacheLastRefreshedAt' | 'type' // GENERATED ALWAYS from parentSlug/grandparentSlug + > + > diff --git a/services/libs/types/src/index.ts b/services/libs/types/src/index.ts index 6563b0e790..1e1b6095cb 100644 --- a/services/libs/types/src/index.ts +++ b/services/libs/types/src/index.ts @@ -22,6 +22,8 @@ export * from './caching' export * from './merging' +export * from './db' + export * from './errors' export * from './pubsub' diff --git a/services/libs/types/src/members.ts b/services/libs/types/src/members.ts index fc31cacdf4..a823bc06cb 100644 --- a/services/libs/types/src/members.ts +++ b/services/libs/types/src/members.ts @@ -227,6 +227,7 @@ export interface IMemberOpensearch { } export interface IChangeAffiliationOverrideData { + id?: string allowAffiliation?: boolean isPrimaryWorkExperience?: boolean memberOrganizationId: string diff --git a/services/libs/types/src/organizations.ts b/services/libs/types/src/organizations.ts index 85674161f8..e8d60fad14 100644 --- a/services/libs/types/src/organizations.ts +++ b/services/libs/types/src/organizations.ts @@ -142,17 +142,6 @@ export interface IOrganizationSyncRemoteData { lastSyncedAt?: string } -export interface NewOrganizationIdentity { - organizationId: string - platform: string - value: string - type: OrganizationIdentityType - verified: boolean - source: string - sourceId?: string | null - integrationId?: string | null -} - export interface IOrganizationIdentity { organizationId?: string platform: string