diff --git a/docs/config-app.md b/docs/config-app.md index 5df4cb13239..015ac1f398e 100644 --- a/docs/config-app.md +++ b/docs/config-app.md @@ -457,6 +457,9 @@ If not defined in config all other Health Checkers would be disabled and endpoin - `gdpr.special-features.sfN.vendor-exceptions[]` - bidder names that will be treated opposite to `sfN.enforce` value. - `gdpr.purpose-one-treatment-interpretation` - option that allows to skip the Purpose one enforcement workflow. - `gdpr.vendorlist.default-timeout-ms` - default operation timeout for obtaining new vendor list. +- `gdpr.vendorlist.live.url` - URL of the latest TCF GVL used to detect vendors with a past `deletedDate`. Default `https://vendor-list.consensu.org/v3/vendor-list.json`. +- `gdpr.vendorlist.live.refresh-period-ms` - how often to refresh the live GVL deleted-vendor set, in milliseconds. Default `86400000` (24 hours). +- `gdpr.vendorlist.live.startup-cache-dir` - directory for local storage vendor list cache. At startup server will attempt to read latest GVL from this location. - `gdpr.vendorlist.v2.http-endpoint-template` - template string for vendor list url version 2. - `gdpr.vendorlist.v2.refresh-missing-list-period-ms` - time to wait between attempts to fetch vendor list version that previously was reported to be missing by origin. Default `3600000` (one hour). - `gdpr.vendorlist.v2.fallback-vendor-list-path` - location on the file system of the fallback vendor list that will be used in place of missing vendor list versions. Optional. diff --git a/docs/metrics.md b/docs/metrics.md index c07e0660598..4dad9b6effb 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -131,6 +131,7 @@ Following metrics are collected and submitted if account is configured with `det - `privacy.tcf.(v1,v2).in-geo` - number of requests received from TCF-concerned geo region with consent string of particular version - `privacy.tcf.(v1,v2).out-geo` - number of requests received outside of TCF-concerned geo region with consent string of particular version - `privacy.tcf.(v1,v2).vendorlist.(missing|ok|err|fallback)` - number of processed vendor lists of particular version +- `privacy.tcf.vendorlist.live.(ok|err)` - number of successful or failed refreshes of the live GVL used for deleted-vendor detection - `privacy.usp.specified` - number of requests with a valid US Privacy string (CCPA) - `privacy.usp.opt-out` - number of requests that required privacy enforcement according to CCPA rules - `privacy.lmt` - number of requests that required privacy enforcement according to LMT flag diff --git a/sample/001_banner/configs/config.yaml b/sample/001_banner/configs/config.yaml index 1dd053ddb22..ca3496226c2 100755 --- a/sample/001_banner/configs/config.yaml +++ b/sample/001_banner/configs/config.yaml @@ -25,6 +25,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/localdev-config.yaml b/sample/configs/localdev-config.yaml index c2991cdf3b6..9c036f3fdb9 100644 --- a/sample/configs/localdev-config.yaml +++ b/sample/configs/localdev-config.yaml @@ -24,3 +24,5 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 diff --git a/sample/configs/prebid-config-db.yaml b/sample/configs/prebid-config-db.yaml index 418b6d365df..666ffb69a55 100644 --- a/sample/configs/prebid-config-db.yaml +++ b/sample/configs/prebid-config-db.yaml @@ -43,6 +43,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/prebid-config-s3.yaml b/sample/configs/prebid-config-s3.yaml index 2a01d4f4d43..09620ceec87 100644 --- a/sample/configs/prebid-config-s3.yaml +++ b/sample/configs/prebid-config-s3.yaml @@ -51,6 +51,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: diff --git a/sample/configs/prebid-config-with-51d-dd.yaml b/sample/configs/prebid-config-with-51d-dd.yaml index 80f73b1a461..1e29ec1d997 100644 --- a/sample/configs/prebid-config-with-51d-dd.yaml +++ b/sample/configs/prebid-config-with-51d-dd.yaml @@ -33,6 +33,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/prebid-config-with-module.yaml b/sample/configs/prebid-config-with-module.yaml index 3619e1cb9fd..959a4cef2af 100644 --- a/sample/configs/prebid-config-with-module.yaml +++ b/sample/configs/prebid-config-with-module.yaml @@ -33,6 +33,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/prebid-config-with-optable.yaml b/sample/configs/prebid-config-with-optable.yaml index 4070cc0aed5..ef2d59037bf 100644 --- a/sample/configs/prebid-config-with-optable.yaml +++ b/sample/configs/prebid-config-with-optable.yaml @@ -39,6 +39,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/prebid-config-with-wurfl.yaml b/sample/configs/prebid-config-with-wurfl.yaml index 55ccd4ffded..9a82d459195 100644 --- a/sample/configs/prebid-config-with-wurfl.yaml +++ b/sample/configs/prebid-config-with-wurfl.yaml @@ -34,6 +34,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/prebid-config.yaml b/sample/configs/prebid-config.yaml index a964c645cc7..f969f25bd60 100644 --- a/sample/configs/prebid-config.yaml +++ b/sample/configs/prebid-config.yaml @@ -34,6 +34,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/src/main/docker/application.yaml b/src/main/docker/application.yaml index 9765fc7ed39..a5e51ed5b4c 100644 --- a/src/main/docker/application.yaml +++ b/src/main/docker/application.yaml @@ -4,3 +4,5 @@ gdpr: cache-dir: /app/prebid-server/data/vendorlist-v2 v3: cache-dir: /app/prebid-server/data/vendorlist-v3 + live: + startup-cache-dir: /app/prebid-server/data/vendorlist-v3 diff --git a/src/main/java/org/prebid/server/metric/Metrics.java b/src/main/java/org/prebid/server/metric/Metrics.java index 372edb9a7df..c6e3394c807 100644 --- a/src/main/java/org/prebid/server/metric/Metrics.java +++ b/src/main/java/org/prebid/server/metric/Metrics.java @@ -554,6 +554,14 @@ public void updatePrivacyTcfVendorListFallbackMetric(int version) { updatePrivacyTcfVendorListMetric(version, MetricName.fallback); } + public void updatePrivacyTcfLiveVendorListOkMetric() { + privacy().tcf().liveVendorList().incCounter(MetricName.ok); + } + + public void updatePrivacyTcfLiveVendorListErrorMetric() { + privacy().tcf().liveVendorList().incCounter(MetricName.err); + } + private void updatePrivacyTcfVendorListMetric(int version, MetricName metricName) { final TcfMetrics tcfMetrics = privacy().tcf(); tcfMetrics.fromVersion(version).vendorList().incCounter(metricName); diff --git a/src/main/java/org/prebid/server/metric/TcfMetrics.java b/src/main/java/org/prebid/server/metric/TcfMetrics.java index 9fd5a811562..72c9a5cd4f6 100644 --- a/src/main/java/org/prebid/server/metric/TcfMetrics.java +++ b/src/main/java/org/prebid/server/metric/TcfMetrics.java @@ -16,6 +16,7 @@ class TcfMetrics extends UpdatableMetrics { private final TcfVersionMetrics tcfVersion1Metrics; private final TcfVersionMetrics tcfVersion2Metrics; + private final LiveVendorListMetrics liveVendorListMetrics; TcfMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { super( @@ -25,6 +26,7 @@ class TcfMetrics extends UpdatableMetrics { tcfVersion1Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v1"); tcfVersion2Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v2"); + liveVendorListMetrics = new LiveVendorListMetrics(metricRegistry, counterType, createTcfPrefix(prefix)); } TcfVersionMetrics fromVersion(int version) { @@ -35,6 +37,10 @@ TcfVersionMetrics fromVersion(int version) { }; } + LiveVendorListMetrics liveVendorList() { + return liveVendorListMetrics; + } + private static String createTcfPrefix(String prefix) { return prefix + ".tcf"; } @@ -87,4 +93,22 @@ private static Function nameCreator(String prefix) { return metricName -> "%s.%s".formatted(prefix, metricName); } } + + static class LiveVendorListMetrics extends UpdatableMetrics { + + LiveVendorListMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { + super( + metricRegistry, + counterType, + nameCreator(createLatestPrefix(prefix))); + } + + private static String createLatestPrefix(String prefix) { + return prefix + ".vendorlist.live"; + } + + private static Function nameCreator(String prefix) { + return metricName -> "%s.%s".formatted(prefix, metricName); + } + } } diff --git a/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java b/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java index 8d47b739426..6010ad950e2 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java @@ -12,6 +12,7 @@ import org.prebid.server.privacy.gdpr.model.VendorPermissionWithGvl; import org.prebid.server.privacy.gdpr.tcfstrategies.purpose.PurposeStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesStrategy; +import org.prebid.server.privacy.gdpr.vendorlist.VendorListWrapper; import org.prebid.server.privacy.gdpr.vendorlist.VersionedVendorListService; import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode; import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; @@ -114,7 +115,7 @@ private Future> permissionsForInternal(Collection processDowngradedSupportedPurposeStrategies( tcfConsent, - wrapWithGVL(vendorPermissionsByType, Collections.emptyMap()), + wrapWithGVL(vendorPermissionsByType, VendorListWrapper.EMPTY), mergedPurposes, mergedPurposeOneTreatmentInterpretation)) .map(ignored -> enforcePurpose4IfRequired(mergedPurposes, vendorPermissionsByType)) @@ -146,7 +147,7 @@ private static VendorPermissionsByType toVendorPermissionsByTy private static VendorPermissionsByType wrapWithGVL( VendorPermissionsByType vendorPermissionsByType, - Map vendorGvlPermissions) { + VendorListWrapper vendorGvlPermissions) { final List weakPermissions = vendorPermissionsByType.getWeakPermissions().stream() .map(vendorPermission -> wrapWithGVL(vendorPermission, vendorGvlPermissions)) @@ -161,7 +162,7 @@ private static VendorPermissionsByType wrapWithGVL( } private static VendorPermissionWithGvl wrapWithGVL(VendorPermission vendorPermission, - Map vendorGvlPermissions) { + VendorListWrapper vendorGvlPermissions) { final Integer vendorId = vendorPermission.getVendorId(); final Vendor vendorGvlByVendorId = Optional.ofNullable(vendorId) diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java new file mode 100644 index 00000000000..49fb8e4e6cb --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java @@ -0,0 +1,127 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import lombok.Getter; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.json.JacksonMapper; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; +import org.prebid.server.metric.Metrics; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; +import org.prebid.server.util.HttpUtil; +import org.prebid.server.vertx.Initializable; +import org.prebid.server.vertx.httpclient.HttpClient; +import org.prebid.server.vertx.httpclient.model.HttpClientResponse; + +import java.time.Clock; +import java.time.Instant; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +public class LiveVendorListService implements Initializable { + + private static final Logger logger = LoggerFactory.getLogger(LiveVendorListService.class); + + private final String cacheDir; + private final String liveGvlUrl; + private final long refreshPeriodMs; + private final int defaultTimeoutMs; + private final Vertx vertx; + private final HttpClient httpClient; + private final VendorListFileStore vendorListFileStore; + private final Metrics metrics; + private final JacksonMapper mapper; + private final Clock clock; + + @Getter + private volatile Set deletedVendorIds = Set.of(); + + public LiveVendorListService(String cacheDir, + String liveGvlUrl, + long refreshPeriodMs, + int defaultTimeoutMs, + Vertx vertx, + HttpClient httpClient, + VendorListFileStore vendorListFileStore, + Metrics metrics, + JacksonMapper mapper, + Clock clock) { + + this.cacheDir = Objects.requireNonNull(cacheDir); + this.liveGvlUrl = HttpUtil.validateUrl(Objects.requireNonNull(liveGvlUrl)); + this.refreshPeriodMs = refreshPeriodMs; + this.defaultTimeoutMs = defaultTimeoutMs; + this.vertx = Objects.requireNonNull(vertx); + this.httpClient = Objects.requireNonNull(httpClient); + this.vendorListFileStore = Objects.requireNonNull(vendorListFileStore); + this.metrics = Objects.requireNonNull(metrics); + this.mapper = Objects.requireNonNull(mapper); + this.clock = Objects.requireNonNull(clock); + } + + @Override + public Future initialize() { + initializeWithLatestCachedVersion(); + vertx.setPeriodic(0, refreshPeriodMs, ignored -> refresh()); + return Future.succeededFuture(); + } + + private void initializeWithLatestCachedVersion() { + vendorListFileStore.getLatestVendorListFromCache(cacheDir).ifPresent(vendorList -> { + saveDeletedVendorsFromVendorList(vendorList); + logger.info("Initialized live GVL from cache with version %d".formatted(vendorList.getVendorListVersion())); + }); + } + + void refresh() { + httpClient.get(liveGvlUrl, defaultTimeoutMs) + .map(this::processResponse) + .map(this::saveDeletedVendorsFromVendorList) + .otherwise(this::handleError); + } + + private Void saveDeletedVendorsFromVendorList(VendorList vendorList) { + updateDeletedVendorIds(extractDeletedVendorIds(vendorList)); + return null; + } + + private VendorList processResponse(HttpClientResponse response) { + final int statusCode = response.getStatusCode(); + if (statusCode != 200) { + throw new PreBidException("HTTP status code " + statusCode); + } + + final String body = response.getBody(); + final VendorList vendorList = VendorListUtil.parseVendorList(body, mapper); + + if (!VendorListUtil.vendorListIsValid(vendorList)) { + throw new PreBidException("Fetched vendor list parsed but has invalid data: " + body); + } + + return vendorList; + } + + Set extractDeletedVendorIds(VendorList vendorList) { + final Instant now = clock.instant(); + return vendorList.getVendors().values().stream() + .filter(vendor -> VendorListUtil.vendorIsDeletedAt(vendor, now)) + .map(Vendor::getId) + .filter(Objects::nonNull) + .collect(Collectors.toUnmodifiableSet()); + } + + private Void updateDeletedVendorIds(Set ids) { + deletedVendorIds = ids; + metrics.updatePrivacyTcfLiveVendorListOkMetric(); + return null; + } + + private Void handleError(Throwable exception) { + logger.warn("Error occurred while fetching live GVL", exception); + metrics.updatePrivacyTcfLiveVendorListErrorMetric(); + return null; + } +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java new file mode 100644 index 00000000000..e02669c8e26 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java @@ -0,0 +1,124 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import io.vertx.core.Future; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.file.FileProps; +import io.vertx.core.file.FileSystem; +import io.vertx.core.file.FileSystemException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.json.JacksonMapper; +import org.prebid.server.log.ConditionalLogger; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +public class VendorListFileStore { + + private static final Logger logger = LoggerFactory.getLogger(VendorListFileStore.class); + private static final ConditionalLogger conditionalLogger = new ConditionalLogger(logger); + + private static final String JSON_SUFFIX = ".json"; + + private final double logSamplingRate; + private final FileSystem fileSystem; + private final JacksonMapper mapper; + + public VendorListFileStore(double logSamplingRate, + FileSystem fileSystem, + JacksonMapper mapper) { + + this.logSamplingRate = logSamplingRate; + this.fileSystem = Objects.requireNonNull(fileSystem); + this.mapper = Objects.requireNonNull(mapper); + } + + Map> createCacheFromDisk(String cacheDir) { + createAndCheckWritePermissionsForCacheDir(cacheDir); + final Map versionToFileContent = readFileSystemCache(cacheDir); + + final Map> cache = new ConcurrentHashMap<>(); + + for (Map.Entry versionAndFileContent : versionToFileContent.entrySet()) { + final VendorList vendorList = VendorListUtil.parseVendorList(versionAndFileContent.getValue(), mapper); + + cache.put(versionAndFileContent.getKey(), vendorList.getVendors()); + } + return cache; + } + + private void createAndCheckWritePermissionsForCacheDir(String cacheDir) { + final FileProps props = fileSystem.existsBlocking(cacheDir) ? fileSystem.propsBlocking(cacheDir) : null; + if (props == null || !props.isDirectory()) { + try { + fileSystem.mkdirsBlocking(cacheDir); + } catch (FileSystemException e) { + throw new PreBidException("Cannot create directory: " + cacheDir, e); + } + } else if (!Files.isWritable(Paths.get(cacheDir))) { + throw new PreBidException("No write permissions for directory: " + cacheDir); + } + } + + private Map readFileSystemCache(String cacheDir) { + return fileSystem.readDirBlocking(cacheDir).stream() + .filter(filepath -> filepath.endsWith(JSON_SUFFIX)) + .collect(Collectors.toMap( + VendorListFileStore::parseCachedFileVersion, + filename -> fileSystem.readFileBlocking(filename).toString())); + } + + Optional getLatestVendorListFromCache(String cacheDir) { + createAndCheckWritePermissionsForCacheDir(cacheDir); + return fileSystem.readDirBlocking(cacheDir).stream() + .filter(filepath -> filepath.endsWith(JSON_SUFFIX)) + .max(Comparator.comparing(VendorListFileStore::parseCachedFileVersion)) + .map(fileSystem::readFileBlocking) + .map(Buffer::toString) + .map(content -> VendorListUtil.parseVendorList(content, mapper)); + } + + private static Integer parseCachedFileVersion(String filepath) { + final String filename = new File(filepath).getName(); + final String filenameWithoutExtension = StringUtils.removeEnd(filename, JSON_SUFFIX); + return Integer.valueOf(filenameWithoutExtension); + } + + Future saveToFile(VendorListResult vendorListResult, String cacheDir, String generationVersion) { + final int version = vendorListResult.getVersion(); + final String filepath = new File(cacheDir, version + JSON_SUFFIX).getPath(); + + return fileSystem.writeFile(filepath, Buffer.buffer(vendorListResult.getVendorListAsString())) + .map(vendorListResult) + .onFailure(error -> conditionalLogger.error( + "Could not create new vendor list for version %s.%s, file: %s, trace: %s".formatted( + generationVersion, version, filepath, ExceptionUtils.getStackTrace(error.getCause())), + logSamplingRate)); + } + + Map readFallbackVendorList(String fallbackVendorListPath) { + if (StringUtils.isBlank(fallbackVendorListPath)) { + return null; + } + + final String vendorListContent = fileSystem.readFileBlocking(fallbackVendorListPath).toString(); + final VendorList vendorList = VendorListUtil.parseVendorList(vendorListContent, mapper); + if (!VendorListUtil.vendorListIsValid(vendorList)) { + throw new PreBidException("Fallback vendor list parsed but has invalid data: " + vendorListContent); + } + + return vendorList.getVendors(); + } +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListResult.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListResult.java new file mode 100644 index 00000000000..827300f90c2 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListResult.java @@ -0,0 +1,14 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import lombok.Value; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +@Value(staticConstructor = "of") +class VendorListResult { + + int version; + + String vendorListAsString; + + VendorList vendorList; +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java index f04c491683f..fb5fe23932f 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java @@ -1,16 +1,8 @@ package org.prebid.server.privacy.gdpr.vendorlist; -import com.github.benmanes.caffeine.cache.Caffeine; import io.netty.handler.codec.http.HttpResponseStatus; import io.vertx.core.Future; import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.file.FileProps; -import io.vertx.core.file.FileSystem; -import io.vertx.core.file.FileSystemException; -import lombok.Value; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.prebid.server.exception.PreBidException; import org.prebid.server.json.JacksonMapper; @@ -24,16 +16,10 @@ import org.prebid.server.vertx.httpclient.HttpClient; import org.prebid.server.vertx.httpclient.model.HttpClientResponse; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Collection; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; /** * Works with GDPR Vendor List. @@ -54,7 +40,6 @@ public class VendorListService { private static final int TCF_VERSION = 2; - private static final String JSON_SUFFIX = ".json"; private static final String VERSION_PLACEHOLDER = "VERSION"; private final double logSamplingRate; @@ -64,21 +49,18 @@ public class VendorListService { private final long refreshMissingListPeriodMs; private final boolean deprecated; private final Vertx vertx; - private final FileSystem fileSystem; private final HttpClient httpClient; private final Metrics metrics; private final String generationVersion; - protected final JacksonMapper mapper; + private final VendorListFetchThrottler fetchThrottler; + private final VendorListFileStore vendorListFileStore; + private final JacksonMapper mapper; - /** - * This is memory/performance optimized model slice: - * map of vendor list version -> map of vendor ID -> Vendors - */ + // Map of vendor list version -> map of vendor ID -> Vendors private final Map> cache; private final Map fallbackVendorList; private final Set versionsToFallback; - private final VendorListFetchThrottler fetchThrottler; public VendorListService(double logSamplingRate, String cacheDir, @@ -88,12 +70,12 @@ public VendorListService(double logSamplingRate, boolean deprecated, String fallbackVendorListPath, Vertx vertx, - FileSystem fileSystem, HttpClient httpClient, Metrics metrics, String generationVersion, - JacksonMapper mapper, - VendorListFetchThrottler fetchThrottler) { + VendorListFetchThrottler fetchThrottler, + VendorListFileStore vendorListFileStore, + JacksonMapper mapper) { this.logSamplingRate = logSamplingRate; this.cacheDir = Objects.requireNonNull(cacheDir); @@ -103,17 +85,15 @@ public VendorListService(double logSamplingRate, this.deprecated = deprecated; this.generationVersion = generationVersion; this.vertx = Objects.requireNonNull(vertx); - this.fileSystem = Objects.requireNonNull(fileSystem); this.httpClient = Objects.requireNonNull(httpClient); this.metrics = Objects.requireNonNull(metrics); - this.mapper = Objects.requireNonNull(mapper); this.fetchThrottler = Objects.requireNonNull(fetchThrottler); + this.vendorListFileStore = Objects.requireNonNull(vendorListFileStore); + this.mapper = Objects.requireNonNull(mapper); - createAndCheckWritePermissionsFor(fileSystem, cacheDir); - cache = Objects.requireNonNull(createCache(fileSystem, cacheDir)); + cache = Objects.requireNonNull(vendorListFileStore.createCacheFromDisk(cacheDir)); - fallbackVendorList = StringUtils.isNotBlank(fallbackVendorListPath) - ? readFallbackVendorList(fallbackVendorListPath) : null; + fallbackVendorList = vendorListFileStore.readFallbackVendorList(fallbackVendorListPath); if (deprecated) { validateFallbackVendorListIfDeprecatedVersion(); } @@ -161,50 +141,6 @@ public Future> forVersion(int version) { .formatted(tcf, generationVersion, version)); } - /** - * Creates vendorList object from string content or throw {@link PreBidException}. - */ - private VendorList toVendorList(String content) { - try { - return mapper.mapper().readValue(content, VendorList.class); - } catch (IOException e) { - final String message = "Cannot parse vendor list from: " + content; - - logger.error(message, e); - throw new PreBidException(message, e); - } - } - - /** - * Returns a Map of vendor id to Vendors. - */ - private Map filterVendorIdToVendors(VendorList vendorList) { - return vendorList.getVendors().entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - /** - * Verifies all significant fields of given {@link VendorList} object. - */ - private boolean isValid(VendorList vendorList) { - return vendorList.getVendorListVersion() != null - && vendorList.getLastUpdated() != null - && MapUtils.isNotEmpty(vendorList.getVendors()) - && isValidVendors(vendorList.getVendors().values()); - } - - private static boolean isValidVendors(Collection vendors) { - return vendors.stream() - .allMatch(vendor -> vendor != null - && vendor.getId() != null - && vendor.getPurposes() != null - && vendor.getLegIntPurposes() != null - && vendor.getFlexiblePurposes() != null - && vendor.getSpecialPurposes() != null - && vendor.getFeatures() != null - && vendor.getSpecialFeatures() != null); - } - /** * Returns the version of TCF which {@link VendorListService} implementation deals with. */ @@ -212,62 +148,6 @@ private int getTcfVersion() { return TCF_VERSION; } - /** - * Creates if doesn't exists and checks write permissions for the given directory. - */ - private static void createAndCheckWritePermissionsFor(FileSystem fileSystem, String dir) { - final FileProps props = fileSystem.existsBlocking(dir) ? fileSystem.propsBlocking(dir) : null; - if (props == null || !props.isDirectory()) { - try { - fileSystem.mkdirsBlocking(dir); - } catch (FileSystemException e) { - throw new PreBidException("Cannot create directory: " + dir, e); - } - } else if (!Files.isWritable(Paths.get(dir))) { - throw new PreBidException("No write permissions for directory: " + dir); - } - } - - /** - * Creates the cache from previously downloaded vendor lists. - */ - private Map> createCache(FileSystem fileSystem, String cacheDir) { - final Map versionToFileContent = readFileSystemCache(fileSystem, cacheDir); - - final Map> cache = Caffeine.newBuilder() - .>build() - .asMap(); - - for (Map.Entry versionAndFileContent : versionToFileContent.entrySet()) { - final VendorList vendorList = toVendorList(versionAndFileContent.getValue()); - final Map vendorIdToVendors = filterVendorIdToVendors(vendorList); - - cache.put(Integer.valueOf(versionAndFileContent.getKey()), vendorIdToVendors); - } - return cache; - } - - /** - * Reads files with .json extension in configured directory and - * returns a {@link Map} where key is a file name without .json extension and value is file content. - */ - private Map readFileSystemCache(FileSystem fileSystem, String dir) { - return fileSystem.readDirBlocking(dir).stream() - .filter(filepath -> filepath.endsWith(JSON_SUFFIX)) - .collect(Collectors.toMap(filepath -> StringUtils.removeEnd(new File(filepath).getName(), JSON_SUFFIX), - filename -> fileSystem.readFileBlocking(filename).toString())); - } - - private Map readFallbackVendorList(String fallbackVendorListPath) { - final String vendorListContent = fileSystem.readFileBlocking(fallbackVendorListPath).toString(); - final VendorList vendorList = toVendorList(vendorListContent); - if (!isValid(vendorList)) { - throw new PreBidException("Fallback vendor list parsed but has invalid data: " + vendorListContent); - } - - return filterVendorIdToVendors(vendorList); - } - private boolean shouldFallback(int version) { return deprecated || (versionsToFallback != null && versionsToFallback.contains(version)); } @@ -290,7 +170,7 @@ private void fetchNewVendorListFor(int version) { * and creates {@link Future} with {@link VendorListResult} from body content * or throws {@link PreBidException} in case of errors. */ - private VendorListResult processResponse(HttpClientResponse response, int version) { + private VendorListResult processResponse(HttpClientResponse response, int version) { final int statusCode = response.getStatusCode(); if (statusCode == HttpResponseStatus.NOT_FOUND.code()) { @@ -301,11 +181,11 @@ private VendorListResult processResponse(HttpClientResponse response } final String body = response.getBody(); - final VendorList vendorList = toVendorList(body); + final VendorList vendorList = VendorListUtil.parseVendorList(body, mapper); // we should care on obtained vendor list, because it'll be saved and never be downloaded again // while application is running - if (!isValid(vendorList)) { + if (!VendorListUtil.vendorListIsValid(vendorList)) { throw new PreBidException("Fetched vendor list parsed but has invalid data: " + body); } @@ -313,25 +193,14 @@ private VendorListResult processResponse(HttpClientResponse response return VendorListResult.of(version, body, vendorList); } - /** - * Saves given vendor list on file system. - */ - private Future> saveToFile(VendorListResult vendorListResult) { - final int version = vendorListResult.getVersion(); - final String filepath = new File(cacheDir, version + JSON_SUFFIX).getPath(); - - return fileSystem.writeFile(filepath, Buffer.buffer(vendorListResult.getVendorListAsString())) - .map(vendorListResult) - .onFailure(error -> conditionalLogger.error( - "Could not create new vendor list for version %s.%s, file: %s, trace: %s".formatted( - generationVersion, version, filepath, ExceptionUtils.getStackTrace(error.getCause())), - logSamplingRate)); + private Future saveToFile(VendorListResult vendorListResult) { + return vendorListFileStore.saveToFile(vendorListResult, cacheDir, generationVersion); } - private Void updateCache(VendorListResult vendorListResult) { + private Void updateCache(VendorListResult vendorListResult) { final int version = vendorListResult.getVersion(); - cache.put(version, filterVendorIdToVendors(vendorListResult.getVendorList())); + cache.put(version, vendorListResult.getVendorList().getVendors()); final int tcf = getTcfVersion(); @@ -387,16 +256,6 @@ private void stopUsingFallbackForVersion(int version) { versionsToFallback.remove(version); } - @Value(staticConstructor = "of") - private static class VendorListResult { - - int version; - - String vendorListAsString; - - T vendorList; - } - private static class MissingVendorListException extends RuntimeException { MissingVendorListException(String message) { diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java new file mode 100644 index 00000000000..371a90e1c55 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java @@ -0,0 +1,56 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import org.apache.commons.collections4.MapUtils; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.json.JacksonMapper; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collection; + +public class VendorListUtil { + + private static final Logger logger = LoggerFactory.getLogger(VendorListUtil.class); + + private VendorListUtil() { + } + + public static VendorList parseVendorList(String content, JacksonMapper mapper) { + try { + return mapper.mapper().readValue(content, VendorList.class); + } catch (IOException e) { + final String message = "Cannot parse vendor list from: " + content; + + logger.error(message, e); + throw new PreBidException(message, e); + } + } + + public static boolean vendorListIsValid(VendorList vendorList) { + return vendorList.getVendorListVersion() != null + && vendorList.getLastUpdated() != null + && MapUtils.isNotEmpty(vendorList.getVendors()) + && vendorsAreValid(vendorList.getVendors().values()); + } + + private static boolean vendorsAreValid(Collection vendors) { + return vendors.stream() + .allMatch(vendor -> vendor != null + && vendor.getId() != null + && vendor.getPurposes() != null + && vendor.getLegIntPurposes() != null + && vendor.getFlexiblePurposes() != null + && vendor.getSpecialPurposes() != null + && vendor.getFeatures() != null + && vendor.getSpecialFeatures() != null); + } + + public static boolean vendorIsDeletedAt(Vendor vendor, Instant now) { + final Instant deletedDate = vendor.getDeletedDate(); + return deletedDate != null && deletedDate.isBefore(now); + } +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListWrapper.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListWrapper.java new file mode 100644 index 00000000000..64ce79352b5 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListWrapper.java @@ -0,0 +1,33 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import lombok.AllArgsConstructor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; + +import java.time.Instant; +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +// The purpose of this wrapper class is to avoid filtering the whole vendor list for deleted vendors +// TODO: move filtering logic to vendor list service +@AllArgsConstructor(staticName = "of") +public class VendorListWrapper { + + public static final VendorListWrapper EMPTY = VendorListWrapper.of( + Collections.emptyMap(), Collections.emptySet(), Instant.EPOCH); + + private final Map vendorList; + private final Set deletedVendorIds; + private final Instant timestamp; + + public Vendor get(Integer key) { + final Vendor vendor = vendorList.get(key); + return isRetained(key, vendor) ? vendor : null; + } + + private boolean isRetained(Integer id, Vendor vendor) { + return vendor != null + && !VendorListUtil.vendorIsDeletedAt(vendor, timestamp) + && !deletedVendorIds.contains(id); + } +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java index 5e261d9b6b4..0f0a63073ca 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java @@ -4,6 +4,7 @@ import io.vertx.core.Future; import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import java.time.Clock; import java.util.Map; import java.util.Objects; @@ -11,18 +12,29 @@ public class VersionedVendorListService { private final VendorListService vendorListServiceV2; private final VendorListService vendorListServiceV3; + private final LiveVendorListService liveVendorListService; + private final Clock clock; + + public VersionedVendorListService(VendorListService vendorListServiceV2, + VendorListService vendorListServiceV3, + LiveVendorListService liveVendorListService, + Clock clock) { - public VersionedVendorListService(VendorListService vendorListServiceV2, VendorListService vendorListServiceV3) { this.vendorListServiceV2 = Objects.requireNonNull(vendorListServiceV2); this.vendorListServiceV3 = Objects.requireNonNull(vendorListServiceV3); + this.liveVendorListService = Objects.requireNonNull(liveVendorListService); + this.clock = Objects.requireNonNull(clock); } - public Future> forConsent(TCString consent) { + public Future forConsent(TCString consent) { final int tcfPolicyVersion = consent.getTcfPolicyVersion(); final int vendorListVersion = consent.getVendorListVersion(); - return tcfPolicyVersion < 4 + final Future> vendorListFuture = tcfPolicyVersion < 4 ? vendorListServiceV2.forVersion(vendorListVersion) : vendorListServiceV3.forVersion(vendorListVersion); + + return vendorListFuture.map(vendors -> + VendorListWrapper.of(vendors, liveVendorListService.getDeletedVendorIds(), clock.instant())); } } diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java index 6bb2be9dddb..336a5d4cdae 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java @@ -6,6 +6,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +import java.time.Instant; import java.util.EnumSet; @AllArgsConstructor @@ -16,6 +17,9 @@ public class Vendor { Integer id; + @JsonProperty("deletedDate") + Instant deletedDate; + EnumSet purposes; @JsonProperty("legIntPurposes") diff --git a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java index b8175828c3b..579dbda2fa6 100644 --- a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java @@ -37,7 +37,9 @@ import org.prebid.server.privacy.gdpr.tcfstrategies.purpose.typestrategies.PurposeTwoBasicEnforcePurposeStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesOneStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesStrategy; +import org.prebid.server.privacy.gdpr.vendorlist.LiveVendorListService; import org.prebid.server.privacy.gdpr.vendorlist.VendorListFetchThrottler; +import org.prebid.server.privacy.gdpr.vendorlist.VendorListFileStore; import org.prebid.server.privacy.gdpr.vendorlist.VendorListService; import org.prebid.server.privacy.gdpr.vendorlist.VersionedVendorListService; import org.prebid.server.settings.model.GdprConfig; @@ -65,6 +67,15 @@ @Configuration public class PrivacyServiceConfiguration { + @Bean + VendorListFileStore vendorListFileStore( + @Value("${logging.sampling-rate:0.01}") double logSamplingRate, + FileSystem fileSystem, + JacksonMapper mapper) { + + return new VendorListFileStore(logSamplingRate, fileSystem, mapper); + } + @Bean VendorListService vendorListServiceV2( @Value("${logging.sampling-rate:0.01}") double logSamplingRate, @@ -72,9 +83,9 @@ VendorListService vendorListServiceV2( VendorListServiceConfigurationProperties vendorListServiceV2Properties, Vertx vertx, Clock clock, - FileSystem fileSystem, HttpClient httpClient, Metrics metrics, + VendorListFileStore vendorListFileStore, JacksonMapper mapper) { return new VendorListService( @@ -86,12 +97,12 @@ VendorListService vendorListServiceV2( vendorListServiceV2Properties.getDeprecated(), vendorListServiceV2Properties.getFallbackVendorListPath(), vertx, - fileSystem, httpClient, metrics, "v2", - mapper, - new VendorListFetchThrottler(vendorListServiceV2Properties.getRetryPolicy().toPolicy(), clock)); + new VendorListFetchThrottler(vendorListServiceV2Properties.getRetryPolicy().toPolicy(), clock), + vendorListFileStore, + mapper); } @Bean @@ -107,9 +118,9 @@ VendorListService vendorListServiceV3( VendorListServiceConfigurationProperties vendorListServiceV3Properties, Vertx vertx, Clock clock, - FileSystem fileSystem, HttpClient httpClient, Metrics metrics, + VendorListFileStore vendorListFileStore, JacksonMapper mapper) { return new VendorListService( @@ -121,12 +132,12 @@ VendorListService vendorListServiceV3( vendorListServiceV3Properties.getDeprecated(), vendorListServiceV3Properties.getFallbackVendorListPath(), vertx, - fileSystem, httpClient, metrics, "v3", - mapper, - new VendorListFetchThrottler(vendorListServiceV3Properties.getRetryPolicy().toPolicy(), clock)); + new VendorListFetchThrottler(vendorListServiceV3Properties.getRetryPolicy().toPolicy(), clock), + vendorListFileStore, + mapper); } @Bean @@ -135,11 +146,40 @@ VendorListServiceConfigurationProperties vendorListServiceV3Properties() { return new VendorListServiceConfigurationProperties(); } + @Bean + LiveVendorListService liveVendorListService( + @Value("${gdpr.vendorlist.live.startup-cache-dir}") String startupCacheDir, + @Value("${gdpr.vendorlist.live.url}") String liveGvlUrl, + @Value("${gdpr.vendorlist.live.refresh-period-ms}") long refreshPeriodMs, + @Value("${gdpr.vendorlist.default-timeout-ms}") int defaultTimeoutMs, + Vertx vertx, + HttpClient httpClient, + VendorListFileStore vendorListFileStore, + Metrics metrics, + JacksonMapper mapper, + Clock clock) { + + return new LiveVendorListService( + startupCacheDir, + liveGvlUrl, + refreshPeriodMs, + defaultTimeoutMs, + vertx, + httpClient, + vendorListFileStore, + metrics, + mapper, + clock); + } + @Bean VersionedVendorListService versionedVendorListService(VendorListService vendorListServiceV2, - VendorListService vendorListServiceV3) { + VendorListService vendorListServiceV3, + LiveVendorListService liveVendorListService, + Clock clock) { - return new VersionedVendorListService(vendorListServiceV2, vendorListServiceV3); + return new VersionedVendorListService( + vendorListServiceV2, vendorListServiceV3, liveVendorListService, clock); } @Bean diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 1422ed73353..4439533ddc7 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -229,6 +229,9 @@ gdpr: max-delay-millis: 120000 factor: 1.1 jitter: 0.2 + live: + url: https://vendor-list.consensu.org/v3/vendor-list.json + refresh-period-ms: 86400000 purposes: p1: enforce-purpose: full diff --git a/src/main/resources/metrics-config/prometheus-labels.yaml b/src/main/resources/metrics-config/prometheus-labels.yaml index b40139d6a8a..162be432f9d 100644 --- a/src/main/resources/metrics-config/prometheus-labels.yaml +++ b/src/main/resources/metrics-config/prometheus-labels.yaml @@ -85,6 +85,10 @@ mappers: labels: tcf: ${0} status: ${1} + - match: privacy.tcf.vendorlist.live.* + name: privacy.tcf.vendorlist.live + labels: + status: ${0} - match: privacy.tcf.*.* name: privacy.tcf.${1} labels: diff --git a/src/test/groovy/org/prebid/server/functional/model/mock/services/vendorlist/VendorListResponse.groovy b/src/test/groovy/org/prebid/server/functional/model/mock/services/vendorlist/VendorListResponse.groovy index d44755978e4..9e54f4ce7e6 100644 --- a/src/test/groovy/org/prebid/server/functional/model/mock/services/vendorlist/VendorListResponse.groovy +++ b/src/test/groovy/org/prebid/server/functional/model/mock/services/vendorlist/VendorListResponse.groovy @@ -34,6 +34,7 @@ class VendorListResponse { List features List specialFeatures String policyUrl + ZonedDateTime deletedDate Overflow overflow String cookieMaxAgeSeconds Boolean usesCookies @@ -41,9 +42,9 @@ class VendorListResponse { Boolean usesNonCookieAccess Boolean deviceStorageDisclosureUrl - static Vendor getDefaultVendor(int id) { + static Vendor getDefaultVendor(int vendorId) { new Vendor().tap { - it.id = id + it.id = vendorId it.name = PBSUtils.randomString it.purposes = [1, 3, 4, 5] it.legIntPurposes = [2, 7, 10] diff --git a/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy b/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy index 249d6fa3f13..2e0d0765a0f 100644 --- a/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy +++ b/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy @@ -448,6 +448,15 @@ class PrebidServerService implements ObjectMapperWrapper { } } + Boolean isMetricFilled(String metricName) { + try { + PBSUtils.waitUntil({ this.sendCollectedMetricsRequest()[metricName] != 0 }) + true + } catch (IllegalStateException ignored) { + false + } + } + Boolean isContainMetricByValue(String value) { try { PBSUtils.waitUntil({ sendInfluxMetricsRequest()[value] != null }) diff --git a/src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/VendorList.groovy b/src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/VendorList.groovy index 343a118f53a..29b0060af27 100644 --- a/src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/VendorList.groovy +++ b/src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/VendorList.groovy @@ -4,7 +4,7 @@ import org.mockserver.matchers.TimeToLive import org.mockserver.matchers.Times import org.mockserver.model.Delay import org.mockserver.model.HttpRequest -import org.mockserver.model.HttpResponse +import org.prebid.server.functional.util.privacy.TcfConsent import org.testcontainers.containers.MockServerContainer import static org.mockserver.model.HttpRequest.request @@ -20,10 +20,13 @@ import static org.prebid.server.functional.util.privacy.TcfConsent.TcfPolicyVers class VendorList extends NetworkScaffolding { - private static final String VENDOR_LIST_ENDPOINT = "/v{TCF_POLICY}/vendor-list.json" + private static final String ENDPOINT_TEMPLATE = "/v{TCF_POLICY}/{FILE_NAME}" - VendorList(MockServerContainer mockServerContainer) { - super(mockServerContainer, VENDOR_LIST_ENDPOINT) + private final String fileName + + VendorList(MockServerContainer mockServerContainer, String fileName = "vendor-list.json") { + super(mockServerContainer, ENDPOINT_TEMPLATE.replace("{FILE_NAME}", fileName)) + this.fileName = fileName } @Override @@ -33,29 +36,40 @@ class VendorList extends NetworkScaffolding { @Override protected HttpRequest getRequest() { - request().withPath(VENDOR_LIST_ENDPOINT) + request().withPath(endpoint) } @Override void reset() { - TcfPolicyVersion.values().each { version -> super.reset("/v${version.vendorListVersion}/vendor-list.json") } + TcfPolicyVersion.values().each { + super.reset("/v${it.vendorListVersion}/${fileName}") + } } void setResponse(TcfPolicyVersion tcfPolicyVersion = TCF_POLICY_V2, Delay delay = null, - Map vendors = [(GENERIC_VENDOR_ID): Vendor.getDefaultVendor(GENERIC_VENDOR_ID)]) { + Map vendors = [(GENERIC_VENDOR_ID): Vendor.getDefaultVendor(GENERIC_VENDOR_ID)], + Integer vendorListVersion = TcfConsent.VENDOR_LIST_VERSION, + Times times = Times.unlimited()) { + def prepareEndpoint = endpoint.replace("{TCF_POLICY}", tcfPolicyVersion.vendorListVersion.toString()) def prepareEncodeResponseBody = encode(defaultVendorListResponse.tap { it.tcfPolicyVersion = tcfPolicyVersion.vendorListVersion it.vendors = vendors + it.vendorListVersion = vendorListVersion it.gvlSpecificationVersion = tcfPolicyVersion >= TcfPolicyVersion.TCF_POLICY_V4 ? V3 : V2 }) - mockServerClient.when(request().withPath(prepareEndpoint), Times.unlimited(), TimeToLive.unlimited(), -10) - .respond { request -> - request.withPath(endpoint) - ? response().withStatusCode(OK_200.code()).withDelay(delay).withBody(prepareEncodeResponseBody) - : HttpResponse.notFoundResponse() - } + def mockResponse = response() + .withStatusCode(OK_200.code()) + .withBody(prepareEncodeResponseBody) + + delay?.with { + mockResponse.withDelay(it) + } + + mockServerClient + .when(request().withPath(prepareEndpoint), times, TimeToLive.unlimited(), -10) + .respond(mockResponse) } } diff --git a/src/test/groovy/org/prebid/server/functional/tests/BidderFieldDisplayBehaviorSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/BidderFieldDisplayBehaviorSpec.groovy index bdf0f633626..6acd01d09cd 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/BidderFieldDisplayBehaviorSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/BidderFieldDisplayBehaviorSpec.groovy @@ -21,6 +21,7 @@ import org.prebid.server.functional.model.request.auction.BidderConfigOrtb import org.prebid.server.functional.model.request.auction.ConsentedProvidersSettings import org.prebid.server.functional.model.request.auction.Device import org.prebid.server.functional.model.request.auction.DeviceExt +import org.prebid.server.functional.model.request.auction.DevicePrebid import org.prebid.server.functional.model.request.auction.Dooh import org.prebid.server.functional.model.request.auction.DoohExt import org.prebid.server.functional.model.request.auction.EidPermission @@ -28,9 +29,7 @@ import org.prebid.server.functional.model.request.auction.ExtPrebidBidderConfig import org.prebid.server.functional.model.request.auction.ExtRequestPrebidData import org.prebid.server.functional.model.request.auction.Interstitial import org.prebid.server.functional.model.request.auction.MultiBid -import org.prebid.server.functional.model.request.auction.PaaFormat import org.prebid.server.functional.model.request.auction.PrebidCache -import org.prebid.server.functional.model.request.auction.DevicePrebid import org.prebid.server.functional.model.request.auction.PrebidCurrency import org.prebid.server.functional.model.request.auction.Renderer import org.prebid.server.functional.model.request.auction.RendererData @@ -47,9 +46,9 @@ import org.prebid.server.functional.util.PBSUtils import static org.prebid.server.functional.model.Currency.USD import static org.prebid.server.functional.model.bidder.BidderName.ALIAS +import static org.prebid.server.functional.model.bidder.BidderName.GENERIC import static org.prebid.server.functional.model.bidder.BidderName.OPENX import static org.prebid.server.functional.model.bidder.BidderName.RUBICON -import static org.prebid.server.functional.model.bidder.BidderName.GENERIC import static org.prebid.server.functional.model.mock.services.currencyconversion.CurrencyConversionRatesResponse.defaultConversionRates import static org.prebid.server.functional.model.request.auction.AdjustmentType.MULTIPLIER import static org.prebid.server.functional.model.request.auction.BidAdjustmentMediaType.BANNER @@ -323,20 +322,6 @@ class BidderFieldDisplayBehaviorSpec extends BaseSpec { } } - def "PBS auction should pass ext.prebid.paaformat to bidder request when paaformat specified"() { - given: "Default bid request" - def bidRequest = BidRequest.defaultBidRequest.tap { - ext.prebid.paaFormat = PaaFormat.ORIGINAL - } - - when: "PBS processes auction request" - defaultPbsService.sendAuctionRequest(bidRequest) - - then: "Bidder request should contain paaFormat value same in request" - def bidderRequest = bidder.getBidderRequest(bidRequest.id) - assert bidderRequest.ext.prebid.paaFormat == bidRequest.ext.prebid.paaFormat - } - def "PBS auction shouldn't pass ext.prebid.kvps to bidder request when kvps specified"() { given: "Default bid request with kvps" def bidRequest = BidRequest.defaultBidRequest.tap { diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/PrivacyBaseSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/PrivacyBaseSpec.groovy index 19f9798a9e8..79ed11c5856 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/PrivacyBaseSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/PrivacyBaseSpec.groovy @@ -8,10 +8,8 @@ import org.prebid.server.functional.model.config.AccountDsaConfig import org.prebid.server.functional.model.config.AccountGdprConfig import org.prebid.server.functional.model.config.AccountGppConfig import org.prebid.server.functional.model.config.AccountPrivacyConfig -import org.prebid.server.functional.model.config.Purpose import org.prebid.server.functional.model.db.Account import org.prebid.server.functional.model.mock.services.vendorlist.VendorListResponse -import org.prebid.server.functional.model.privacy.EnforcementRequirement import org.prebid.server.functional.model.privacy.gpp.GppDataActivity import org.prebid.server.functional.model.privacy.gpp.UsCaliforniaV1ChildSensitiveData import org.prebid.server.functional.model.privacy.gpp.UsCaliforniaV1SensitiveData @@ -55,12 +53,9 @@ import org.prebid.server.functional.util.privacy.gpp.v1.UsVaV1Consent import static org.prebid.server.functional.model.bidder.BidderName.GENERIC import static org.prebid.server.functional.model.bidder.BidderName.OPENX -import static org.prebid.server.functional.model.config.PurposeEnforcement.BASIC -import static org.prebid.server.functional.model.config.PurposeEnforcement.FULL -import static org.prebid.server.functional.model.config.PurposeEnforcement.NO import static org.prebid.server.functional.model.mock.services.vendorlist.VendorListResponse.getDefaultVendorListResponse -import static org.prebid.server.functional.model.pricefloors.Country.USA import static org.prebid.server.functional.model.pricefloors.Country.BULGARIA +import static org.prebid.server.functional.model.pricefloors.Country.USA import static org.prebid.server.functional.model.request.GppSectionId.US_CA_V1 import static org.prebid.server.functional.model.request.GppSectionId.US_CO_V1 import static org.prebid.server.functional.model.request.GppSectionId.US_CT_V1 @@ -74,10 +69,6 @@ import static org.prebid.server.functional.model.request.auction.TraceLevel.VERB import static org.prebid.server.functional.model.response.cookiesync.UserSyncInfo.Type.REDIRECT import static org.prebid.server.functional.testcontainers.Dependencies.getNetworkServiceContainer import static org.prebid.server.functional.util.privacy.TcfConsent.GENERIC_VENDOR_ID -import static org.prebid.server.functional.util.privacy.TcfConsent.PurposeId.BASIC_ADS -import static org.prebid.server.functional.util.privacy.TcfConsent.RestrictionType.REQUIRE_CONSENT -import static org.prebid.server.functional.util.privacy.TcfConsent.RestrictionType.REQUIRE_LEGITIMATE_INTEREST -import static org.prebid.server.functional.util.privacy.TcfConsent.RestrictionType.UNDEFINED import static org.prebid.server.functional.util.privacy.TcfConsent.TcfPolicyVersion.TCF_POLICY_V2 import static org.prebid.server.functional.util.privacy.model.State.ALABAMA @@ -89,7 +80,7 @@ abstract class PrivacyBaseSpec extends BaseSpec { "adapters.${GENERIC.value}.ortb-version" : "2.6"] private static final Map OPENX_CONFIG = ["adaptrs.${OPENX.value}.enabled" : "true", "adapters.${OPENX.value}.usersync.cookie-family-name": OPENX.value, - "adapters.${OPENX}.ortb-version" : "2.6", + "adapters.${OPENX.value}.ortb-version" : "2.6", "adapters.${OPENX.value}.endpoint" : "$networkServiceContainer.rootUri/auction".toString(), "adapters.${OPENX.value}.enabled" : 'true'] protected static final Map GDPR_VENDOR_LIST_CONFIG = ["gdpr.vendorlist.v2.http-endpoint-template": "$networkServiceContainer.rootUri/v2/vendor-list.json".toString(), diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy index 21abcb59c6b..545355d3efd 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy @@ -1,5 +1,6 @@ package org.prebid.server.functional.tests.privacy.gdpr +import org.mockserver.matchers.Times import org.mockserver.model.Delay import org.prebid.server.functional.model.ChannelType import org.prebid.server.functional.model.config.AccountGdprConfig @@ -7,25 +8,31 @@ import org.prebid.server.functional.model.config.AccountMetricsConfig import org.prebid.server.functional.model.config.AccountMetricsVerbosityLevel import org.prebid.server.functional.model.config.PurposeConfig import org.prebid.server.functional.model.config.PurposeEnforcement +import org.prebid.server.functional.model.mock.services.vendorlist.VendorListResponse import org.prebid.server.functional.model.pricefloors.Country +import org.prebid.server.functional.model.privacy.EnforcementRequirement import org.prebid.server.functional.model.request.Channel import org.prebid.server.functional.model.request.auction.DistributionChannel import org.prebid.server.functional.model.request.auction.Regs import org.prebid.server.functional.model.request.auction.RegsExt import org.prebid.server.functional.model.response.auction.ErrorType +import org.prebid.server.functional.model.response.auction.NoBidResponse +import org.prebid.server.functional.service.PrebidServerService +import org.prebid.server.functional.testcontainers.scaffolding.VendorList import org.prebid.server.functional.tests.privacy.PrivacyBaseSpec import org.prebid.server.functional.util.PBSUtils import org.prebid.server.functional.util.privacy.BogusConsent import org.prebid.server.functional.util.privacy.TcfConsent +import org.prebid.server.functional.util.privacy.TcfUtils import org.prebid.server.functional.util.privacy.VendorListConsent import spock.lang.PendingFeature import java.time.Instant +import java.time.ZonedDateTime import static org.prebid.server.functional.model.ChannelType.PBJS import static org.prebid.server.functional.model.ChannelType.WEB import static org.prebid.server.functional.model.bidder.BidderName.GENERIC - import static org.prebid.server.functional.model.config.AccountMetricsVerbosityLevel.DETAILED import static org.prebid.server.functional.model.config.Purpose.P1 import static org.prebid.server.functional.model.config.Purpose.P2 @@ -46,6 +53,7 @@ import static org.prebid.server.functional.model.request.auction.PublicCountryIp import static org.prebid.server.functional.model.request.auction.TraceLevel.BASIC import static org.prebid.server.functional.model.request.auction.TraceLevel.VERBOSE import static org.prebid.server.functional.model.response.auction.BidRejectionReason.REQUEST_BLOCKED_PRIVACY +import static org.prebid.server.functional.testcontainers.Dependencies.getNetworkServiceContainer import static org.prebid.server.functional.util.privacy.TcfConsent.GENERIC_VENDOR_ID import static org.prebid.server.functional.util.privacy.TcfConsent.PurposeId.BASIC_ADS import static org.prebid.server.functional.util.privacy.TcfConsent.PurposeId.DEVICE_ACCESS @@ -55,6 +63,21 @@ import static org.prebid.server.functional.util.privacy.TcfConsent.TcfPolicyVers class GdprAuctionSpec extends PrivacyBaseSpec { + private static final String LIVE_GVL_FILE_NAME = "live-gvl-vendor-list.json" + private static final VENDOR_LIST_VERSION = PBSUtils.getRandomNumber(2, 4094) + private static final Map REFRESH_LIVE_VENDOR_LIST_CONFIG = GENERAL_PRIVACY_CONFIG + + ["gdpr.vendorlist.live.url": "$networkServiceContainer.rootUri/v3/$LIVE_GVL_FILE_NAME".toString()] + private static final VendorList liveVendorListResponse = new VendorList(networkServiceContainer, LIVE_GVL_FILE_NAME) + + def setup() { + vendorListResponse.setResponse() + liveVendorListResponse.reset() + } + + def cleanup() { + vendorListResponse.reset() + } + @PendingFeature def "PBS should add debug log for auction request when valid gdpr was passed"() { given: "Default gdpr BidRequest" @@ -1116,4 +1139,261 @@ class GdprAuctionSpec extends PrivacyBaseSpec { assert metrics["adapter.${GENERIC.value}.requests.buyeruid_scrubbed"] == 1 assert metrics["account.${account.uuid}.adapter.${GENERIC.value}.requests.buyeruid_scrubbed"] == 1 } + + def "PBS should treated GVL record as valid when it's don't marked as deleted for GVL file"() { + given: "Vendor list response with not passed delete date for bidder" + def vendorList = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = gvlDeletedDate + } + downloadLiveGvtList(VENDOR_LIST_VERSION, vendorList) + + and: "PBS with proper warmed GVL config" + def pbsWithLiveGvlSetup = pbsServiceFactory.getService(REFRESH_LIVE_VENDOR_LIST_CONFIG) + warmupGvtList(pbsWithLiveGvlSetup, VENDOR_LIST_VERSION) + + and: "Valid tcf full enforcement requirments" + def requirement = EnforcementRequirement.getDefaultFull(GENERIC_VENDOR_ID, VENDOR_LIST_VERSION, P2) + + and: "Tcf consent setup" + def tcfConsent = TcfUtils.getConsentString(requirement) + + and: "Bid request with tcf consent string" + def bidRequest = getGdprBidRequest(tcfConsent) + + and: "Account with tcf config" + def accountGdprConfig = new AccountGdprConfig(purposes: TcfUtils.getPurposeConfigsForPersonalizedAds(requirement)) + def account = getAccountWithGdpr(bidRequest.accountId, accountGdprConfig) + accountDao.save(account) + + and: "Flush metric" + flushMetrics(pbsWithLiveGvlSetup) + + when: "PBS processes auction request" + def response = pbsWithLiveGvlSetup.sendAuctionRequest(bidRequest) + + then: "Response should contain seatBid" + assert response.seatbid.bid.flatten().size() == 1 + + "Response shouldn't contain any nrb, errors and warnings" + assert verifyAll(response) { + !it.noBidResponse + !it.ext.warnings + !it.ext.errors + } + + and: "Bidder request should be valid" + assert bidder.getBidderRequests(bidRequest.id) + + and: "Disallowed metrics shouldn't be updated" + def metrics = pbsWithLiveGvlSetup.sendCollectedMetricsRequest() + assert !metrics[TEMPLATE_ADAPTER_DISALLOWED_COUNT.getValue(bidRequest, FETCH_BIDS)] + + cleanup: "Stop and remove pbs container" + pbsServiceFactory.removeContainer(REFRESH_LIVE_VENDOR_LIST_CONFIG) + + where: + gvlDeletedDate << [null, ZonedDateTime.now().plusYears(1)] + } + + def "PBS should treated GVL record as invalid when it's marked as deleted for GVL file"() { + given: "Vendor list response with passed delete date for bidder" + def vendor = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = liveGvlDeletedDate + } + downloadLiveGvtList(VENDOR_LIST_VERSION, vendor) + + and: "PBS with proper warmed GVL config" + def pbsWithLiveGvlSetup = pbsServiceFactory.getService(REFRESH_LIVE_VENDOR_LIST_CONFIG) + def requestVendor = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = requestGvlDeletedDate + } + warmupGvtList(pbsWithLiveGvlSetup, VENDOR_LIST_VERSION, requestVendor) + + and: "Valid tcf full enforcement requirments" + def requirement = EnforcementRequirement.getDefaultFull(GENERIC_VENDOR_ID, VENDOR_LIST_VERSION, P2) + + and: "Tcf consent setup" + def tcfConsent = TcfUtils.getConsentString(requirement) + + and: "Bid request with tcf consent string" + def bidRequest = getGdprBidRequest(tcfConsent) + + and: "Account with tcf config" + def accountGdprConfig = new AccountGdprConfig(purposes: TcfUtils.getPurposeConfigsForPersonalizedAds(requirement)) + def account = getAccountWithGdpr(bidRequest.accountId, accountGdprConfig) + accountDao.save(account) + + and: "Flush metric" + flushMetrics(pbsWithLiveGvlSetup) + + when: "PBS processes auction request" + def response = pbsWithLiveGvlSetup.sendAuctionRequest(bidRequest) + + then: "Response shouldn't contain any seatBids, errors and warnings" + assert verifyAll(response) { + !it.seatbid.size() + !it.ext.warnings + !it.ext.errors + } + + and: "Response should include nbr" + assert response.noBidResponse == NoBidResponse.UNKNOWN_ERROR + + and: "PBS should cansel request" + assert !bidder.getBidderRequests(bidRequest.id) + + and: "Disallowed metrics should be updated" + def metrics = pbsWithLiveGvlSetup.sendCollectedMetricsRequest() + assert metrics[TEMPLATE_ADAPTER_DISALLOWED_COUNT.getValue(bidRequest, FETCH_BIDS)] == 1 + + cleanup: "Stop and remove pbs container" + pbsServiceFactory.removeContainer(REFRESH_LIVE_VENDOR_LIST_CONFIG) + + where: + liveGvlDeletedDate | requestGvlDeletedDate + ZonedDateTime.now().minusSeconds(1) | ZonedDateTime.now().plusYears(1) + ZonedDateTime.now().plusYears(1) | ZonedDateTime.now().minusSeconds(1) + } + + def "PBS should treated GVL record as invalid when it's marked as deleted for newer GVL file"() { + given: "Newer Vendor list response with passed delete date for bidder" + def vendorList = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = ZonedDateTime.now().minusSeconds(1) + } + downloadLiveGvtList(VENDOR_LIST_VERSION, vendorList) + + and: "PBS with proper warmed older GVL config" + def pbsWithLiveGvlSetup = pbsServiceFactory.getService(REFRESH_LIVE_VENDOR_LIST_CONFIG) + warmupGvtList(pbsWithLiveGvlSetup, VENDOR_LIST_VERSION - 1) + + and: "Valid tcf full enforcement requirments for older gvl list" + def requirement = EnforcementRequirement.getDefaultFull(GENERIC_VENDOR_ID, VENDOR_LIST_VERSION - 1, P2) + + and: "Tcf consent setup" + def tcfConsent = TcfUtils.getConsentString(requirement) + + and: "Bid request with tcf consent string" + def bidRequest = getGdprBidRequest(tcfConsent) + + and: "Account with tcf config" + def accountGdprConfig = new AccountGdprConfig(purposes: TcfUtils.getPurposeConfigsForPersonalizedAds(requirement)) + def account = getAccountWithGdpr(bidRequest.accountId, accountGdprConfig) + accountDao.save(account) + + and: "Flush metric" + flushMetrics(pbsWithLiveGvlSetup) + + when: "PBS processes auction request" + def response = pbsWithLiveGvlSetup.sendAuctionRequest(bidRequest) + + then: "Response shouldn't contain any seatBids, errors and warnings" + assert verifyAll(response) { + !it.seatbid.size() + !it.ext.warnings + !it.ext.errors + } + + and: "Response should include nbr" + assert response.noBidResponse == NoBidResponse.UNKNOWN_ERROR + + and: "PBS should cansel request" + assert !bidder.getBidderRequests(bidRequest.id) + + and: "Disallowed metrics should be updated" + def metrics = pbsWithLiveGvlSetup.sendCollectedMetricsRequest() + assert metrics[TEMPLATE_ADAPTER_DISALLOWED_COUNT.getValue(bidRequest, FETCH_BIDS)] == 1 + + cleanup: "Stop and remove pbs container" + pbsServiceFactory.removeContainer(REFRESH_LIVE_VENDOR_LIST_CONFIG) + } + + def "PBS should treat vendor as valid when deleted only in intermediate GVL versions"() { + given: "Old vendor list response with passed delete date for bidder" + def olderVendorList = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = ZonedDateTime.now().minusSeconds(1) + } + downloadLiveGvtList(VENDOR_LIST_VERSION, olderVendorList, Times.once()) + + and: "PBS with a high-frequency live GVL refresh config" + def config = REFRESH_LIVE_VENDOR_LIST_CONFIG + ['gdpr.vendorlist.live.refresh-period-ms': '1000'] + def pbsWithLiveGvlSetup = pbsServiceFactory.getService(config) + + and: "Flash metrics" + flushMetrics(pbsWithLiveGvlSetup) + + and: "Newer vendor list where bidder is not deleted yet" + def newerVendorList = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = ZonedDateTime.now().plusYears(1) + } + downloadLiveGvtList(VENDOR_LIST_VERSION + 1, newerVendorList, Times.once()) + pbsWithLiveGvlSetup.isContainMetricByValue('privacy.tcf.vendorlist.live.ok') + + and: "GVL list is warmed up for PBS" + warmupGvtList(pbsWithLiveGvlSetup, VENDOR_LIST_VERSION - 1) + + and: "Valid tcf full enforcement requirments for older GVL list" + def requirement = EnforcementRequirement.getDefaultFull(GENERIC_VENDOR_ID, VENDOR_LIST_VERSION - 1, P2) + + and: "Tcf consent setup" + def tcfConsent = TcfUtils.getConsentString(requirement) + + and: "Bid request with tcf consent string" + def bidRequest = getGdprBidRequest(tcfConsent) + + and: "Account with tcf config" + def accountGdprConfig = new AccountGdprConfig(purposes: TcfUtils.getPurposeConfigsForPersonalizedAds(requirement)) + def account = getAccountWithGdpr(bidRequest.accountId, accountGdprConfig) + accountDao.save(account) + + and: "Flush metric" + flushMetrics(pbsWithLiveGvlSetup) + + when: "PBS processes auction request" + def response = pbsWithLiveGvlSetup.sendAuctionRequest(bidRequest) + + then: "Response should contain seatBid" + assert response.seatbid.bid.flatten().size() == 1 + + "Response shouldn't contain any nrb, errors and warnings" + assert verifyAll(response) { + !it.noBidResponse + !it.ext.warnings + !it.ext.errors + } + + and: "Bidder request should be valid" + assert bidder.getBidderRequests(bidRequest.id) + + and: "Disallowed metrics shouldn't be updated" + def metrics = pbsWithLiveGvlSetup.sendCollectedMetricsRequest() + assert metrics[TEMPLATE_ADAPTER_DISALLOWED_COUNT.getValue(bidRequest, FETCH_BIDS)] == 0 + + cleanup: "Stop and remove pbs container" + pbsServiceFactory.removeContainer(config) + } + + private static void downloadLiveGvtList( + Integer vendorListVersion = VENDOR_LIST_VERSION, + VendorListResponse.Vendor vendor = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID), + Times times = Times.unlimited()) { + + liveVendorListResponse.reset() + liveVendorListResponse.setResponse(TCF_POLICY_V5, null, [(vendor.id): vendor], vendorListVersion, times) + } + + private static void warmupGvtList(PrebidServerService pbsService, + Integer vendorListVersion = VENDOR_LIST_VERSION, + VendorListResponse.Vendor vendor = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID)) { + + def simpleTcfString = new TcfConsent.Builder() + .setDisclosedVendors([GENERIC_VENDOR_ID]) + .setVendorListVersion(vendorListVersion) + .build() + + vendorList.reset() + vendorList.setResponse(TCF_POLICY_V2, null, [(vendor.id): vendor]) + + def bidRequest = getGdprBidRequest(simpleTcfString) + pbsService.sendAuctionRequest(bidRequest) + } } diff --git a/src/test/groovy/org/prebid/server/functional/util/privacy/TcfConsent.groovy b/src/test/groovy/org/prebid/server/functional/util/privacy/TcfConsent.groovy index d0d9e19512e..363c7eca4bd 100644 --- a/src/test/groovy/org/prebid/server/functional/util/privacy/TcfConsent.groovy +++ b/src/test/groovy/org/prebid/server/functional/util/privacy/TcfConsent.groovy @@ -12,9 +12,9 @@ import static org.prebid.server.functional.util.privacy.TcfConsent.TcfPolicyVers class TcfConsent implements ConsentString { - public static final Integer RUBICON_VENDOR_ID = PBSUtils.getRandomNumber(0, 65534) - public static final Integer GENERIC_VENDOR_ID = PBSUtils.getRandomNumber(0, 65534) - public static final Integer VENDOR_LIST_VERSION = PBSUtils.getRandomNumber(0, 4095) + public static final Integer RUBICON_VENDOR_ID = PBSUtils.getRandomNumber(1, 65534) + public static final Integer GENERIC_VENDOR_ID = PBSUtils.getRandomNumber(1, 65534) + public static final Integer VENDOR_LIST_VERSION = PBSUtils.getRandomNumber(1, 4095) private final TCStringEncoder.Builder tcStringEncoder diff --git a/src/test/java/org/prebid/server/metric/MetricsTest.java b/src/test/java/org/prebid/server/metric/MetricsTest.java index 8a56e279d30..eb0af781a7d 100644 --- a/src/test/java/org/prebid/server/metric/MetricsTest.java +++ b/src/test/java/org/prebid/server/metric/MetricsTest.java @@ -1004,6 +1004,24 @@ public void updatePrivacyTcfVendorListFallbackMetricShouldIncrementMetric() { assertThat(metricRegistry.counter("privacy.tcf.v1.vendorlist.fallback").getCount()).isEqualTo(1); } + @Test + public void updatePrivacyTcfLiveVendorListOkMetricShouldIncrementMetric() { + // when + metrics.updatePrivacyTcfLiveVendorListOkMetric(); + + // then + assertThat(metricRegistry.counter("privacy.tcf.vendorlist.live.ok").getCount()).isOne(); + } + + @Test + public void updatePrivacyTcfLiveVendorListErrorMetricShouldIncrementMetric() { + // when + metrics.updatePrivacyTcfLiveVendorListErrorMetric(); + + // then + assertThat(metricRegistry.counter("privacy.tcf.vendorlist.live.err").getCount()).isOne(); + } + @Test public void shouldNotUpdateAccountMetricsIfVerbosityIsNone() { // given diff --git a/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java index 19c4d034b2d..bc99140d6db 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java @@ -15,6 +15,7 @@ import org.prebid.server.privacy.gdpr.model.VendorPermissionWithGvl; import org.prebid.server.privacy.gdpr.tcfstrategies.purpose.PurposeStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesStrategy; +import org.prebid.server.privacy.gdpr.vendorlist.VendorListWrapper; import org.prebid.server.privacy.gdpr.vendorlist.VersionedVendorListService; import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; import org.prebid.server.settings.model.AccountGdprConfig; @@ -34,7 +35,6 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static org.apache.commons.collections4.SetUtils.hashSet; @@ -103,7 +103,7 @@ public class Tcf2ServiceTest extends VertxTest { @BeforeEach public void setUp() { - given(vendorListService.forConsent(any())).willReturn(Future.succeededFuture(emptyMap())); + given(vendorListService.forConsent(any())).willReturn(Future.succeededFuture(VendorListWrapper.EMPTY)); given(purposeStrategyOne.getPurpose()).willReturn(ONE); given(purposeStrategyTwo.getPurpose()).willReturn(TWO); diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java new file mode 100644 index 00000000000..863ac4a978b --- /dev/null +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java @@ -0,0 +1,288 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.server.VertxTest; +import org.prebid.server.metric.Metrics; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Feature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialFeature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialPurpose; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; +import org.prebid.server.vertx.httpclient.HttpClient; +import org.prebid.server.vertx.httpclient.model.HttpClientResponse; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Date; +import java.util.EnumSet; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.ONE; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.TWO; + +@ExtendWith(MockitoExtension.class) +public class LiveVendorListServiceTest extends VertxTest { + + private static final Instant NOW = Instant.parse("2024-06-01T12:00:00Z"); + private static final String CACHE_DIR = "/cache/dir"; + private static final String LIVE_GVL_URL = "https://example.com"; + private static final long REFRESH_PERIOD_MS = 1000; + + @Mock + private Vertx vertx; + @Mock + private HttpClient httpClient; + @Mock + private VendorListFileStore vendorListFileStore; + @Mock + private Metrics metrics; + + private LiveVendorListService target; + + @BeforeEach + public void setUp() { + target = new LiveVendorListService( + CACHE_DIR, + LIVE_GVL_URL, + REFRESH_PERIOD_MS, + 1000, + vertx, + httpClient, + vendorListFileStore, + metrics, + jacksonMapper, + Clock.fixed(NOW, ZoneOffset.UTC)); + } + + @Test + public void getDeletedVendorIdsShouldReturnEmptySetWhenFetchNeverSucceeded() { + // when and then + assertThat(target.getDeletedVendorIds()).isEmpty(); + } + + @Test + public void getDeletedVendorIdsShouldReturnVendorThatIsDeletedInLiveVendorList() throws JsonProcessingException { + // given + final Vendor vendor = givenVendor(42, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.succeededFuture(HttpClientResponse.of(200, null, responseBody))); + + // when + target.refresh(); + + // then + assertThat(target.getDeletedVendorIds()).containsExactly(42); + } + + @Test + public void extractDeletedVendorIdsShouldReturnOnlyVendorsWithPastDeletedDate() { + // given + final VendorList vendorList = givenVendorList( + givenVendor(1, "2024-01-01T00:00:00Z"), + givenVendor(2, null), + givenVendor(3, "2025-01-01T00:00:00Z"), + givenVendor(4, "2024-06-01T12:00:00Z")); + + // when + final var deletedIds = target.extractDeletedVendorIds(vendorList); + + // then + assertThat(deletedIds).containsExactly(1); + } + + @Test + public void refreshShouldUpdateDeletedVendorIdsAndIncrementOkMetric() throws JsonProcessingException { + // given + final Vendor vendor = givenVendor(1, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + givenHttpClientReturnsResponse(200, responseBody); + + // when + target.refresh(); + + // then + assertThat(target.getDeletedVendorIds()).containsExactly(1); + verify(metrics).updatePrivacyTcfLiveVendorListOkMetric(); + verify(metrics, never()).updatePrivacyTcfLiveVendorListErrorMetric(); + } + + @Test + public void refreshShouldReplaceDeletedVendorIdsOnSubsequentSuccessfulFetch() throws JsonProcessingException { + // given + final Vendor vendor = givenVendor(1, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + final Vendor vendor2 = givenVendor(2, "2024-02-01T00:00:00Z"); + final String responseBody2 = mapper.writeValueAsString(givenVendorList(vendor2)); + given(httpClient.get(anyString(), anyLong())) + .willReturn( + Future.succeededFuture(HttpClientResponse.of(200, null, responseBody)), + Future.succeededFuture(HttpClientResponse.of(200, null, responseBody2))); + + // when + target.refresh(); + target.refresh(); + + // then + assertThat(target.getDeletedVendorIds()).containsExactly(2); + } + + @Test + public void refreshShouldIncrementErrorMetricOnHttpFailure() { + // given + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.failedFuture(new RuntimeException("connection failed"))); + + // when + target.refresh(); + + // then + assertThat(target.getDeletedVendorIds()).isEmpty(); + verify(metrics).updatePrivacyTcfLiveVendorListErrorMetric(); + verify(metrics, never()).updatePrivacyTcfLiveVendorListOkMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnNonOkStatus() { + // given + givenHttpClientReturnsResponse(503, "{}"); + + // when + target.refresh(); + + // then + assertThat(target.getDeletedVendorIds()).isEmpty(); + verify(metrics).updatePrivacyTcfLiveVendorListErrorMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnInvalidJson() { + // given + givenHttpClientReturnsResponse(200, "invalid-json"); + + // when + target.refresh(); + + // then + assertThat(target.getDeletedVendorIds()).isEmpty(); + verify(metrics).updatePrivacyTcfLiveVendorListErrorMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnInvalidVendorList() throws JsonProcessingException { + // given + final Vendor vendor = givenVendor(42, "2024-01-01T00:00:00Z") + .toBuilder().features(null).build(); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + givenHttpClientReturnsResponse(200, responseBody); + + // when + target.refresh(); + + // then + assertThat(target.getDeletedVendorIds()).isEmpty(); + verify(metrics).updatePrivacyTcfLiveVendorListErrorMetric(); + } + + @Test + public void initializeShouldLoadDeletedVendorsFromCachedVendorList() { + // given + final VendorList vendorList = givenVendorList(givenVendor(42, "2024-01-01T00:00:00Z")); + given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.of(vendorList)); + + // when + target.initialize(); + + // then + assertThat(target.getDeletedVendorIds()).containsExactly(42); + } + + @Test + public void initializeShouldSchedulePeriodicRefresh() { + // given + given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.empty()); + + // when + target.initialize(); + + // then + verify(vertx).setPeriodic(eq(0L), eq(REFRESH_PERIOD_MS), any()); + } + + @Test + public void initializeShouldReturnSucceededFuture() { + // given + given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.empty()); + + // when + final Future result = target.initialize(); + + // then + assertThat(result.succeeded()).isTrue(); + } + + @Test + public void refreshShouldKeepLastGoodSetOnFailureAfterSuccessfulFetch() throws JsonProcessingException { + // given + final Vendor vendor = givenVendor(1, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + given(httpClient.get(anyString(), anyLong())) + .willReturn( + Future.succeededFuture(HttpClientResponse.of(200, null, responseBody)), + Future.failedFuture(new RuntimeException("connection failed"))); + + // when + target.refresh(); + target.refresh(); + + // then + assertThat(target.getDeletedVendorIds()).containsExactly(1); + verify(metrics).updatePrivacyTcfLiveVendorListOkMetric(); + verify(metrics).updatePrivacyTcfLiveVendorListErrorMetric(); + } + + private void givenHttpClientReturnsResponse(int statusCode, String response) { + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.succeededFuture(HttpClientResponse.of(statusCode, null, response))); + } + + private static Vendor givenVendor(int id, String deletedDate) { + return Vendor.builder() + .id(id) + .deletedDate(deletedDate != null ? Instant.parse(deletedDate) : null) + .purposes(EnumSet.of(ONE)) + .legIntPurposes(EnumSet.of(TWO)) + .flexiblePurposes(EnumSet.noneOf(PurposeCode.class)) + .specialPurposes(EnumSet.noneOf(SpecialPurpose.class)) + .features(EnumSet.noneOf(Feature.class)) + .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) + .build(); + } + + private static VendorList givenVendorList(Vendor... vendors) { + return VendorList.of( + 1, + Date.from(Instant.parse("2020-08-20T16:05:24Z")), + Arrays.stream(vendors).collect(Collectors.toMap(Vendor::getId, Function.identity()))); + } +} diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java new file mode 100644 index 00000000000..dea4c1ce67c --- /dev/null +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java @@ -0,0 +1,407 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.vertx.core.Future; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.file.FileProps; +import io.vertx.core.file.FileSystem; +import io.vertx.core.file.FileSystemException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.server.VertxTest; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Feature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialFeature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialPurpose; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.io.File; +import java.nio.file.Path; +import java.util.Date; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.ONE; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.TWO; + +@ExtendWith(MockitoExtension.class) +public class VendorListFileStoreTest extends VertxTest { + + private static final String CACHE_DIR = "/cache/dir"; + private static final String FALLBACK_VENDOR_LIST_PATH = "fallback.json"; + private static final String GENERATION_VERSION = "v0"; + + @Mock + private FileSystem fileSystem; + + @TempDir + private Path tempDir; + + private VendorListFileStore target; + + @BeforeEach + public void setUp() { + target = new VendorListFileStore(0, fileSystem, jacksonMapper); + } + + @Test + public void createCacheFromDiskShouldCreateCacheDirWhenItDoesNotExist() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + target.createCacheFromDisk(CACHE_DIR); + + // then + verify(fileSystem).mkdirsBlocking(eq(CACHE_DIR)); + } + + @Test + public void createCacheFromDiskShouldCreateCacheDirWhenPathExistsButIsNotADirectory() { + // given + final FileProps fileProps = mock(FileProps.class); + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(true); + given(fileSystem.propsBlocking(eq(CACHE_DIR))).willReturn(fileProps); + given(fileProps.isDirectory()).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + target.createCacheFromDisk(CACHE_DIR); + + // then + verify(fileSystem).mkdirsBlocking(eq(CACHE_DIR)); + } + + @Test + public void createCacheFromDiskShouldFailIfCannotCreateCacheDir() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.mkdirsBlocking(eq(CACHE_DIR))) + .willThrow(new FileSystemException("dir creation error")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot create directory: " + CACHE_DIR); + } + + @Test + public void createCacheFromDiskShouldFailIfCacheDirIsNotWritable() { + // given + final String cacheDir = tempDir.toString(); + tempDir.toFile().setWritable(false, false); + final FileProps fileProps = mock(FileProps.class); + given(fileSystem.existsBlocking(eq(cacheDir))).willReturn(true); + given(fileSystem.propsBlocking(eq(cacheDir))).willReturn(fileProps); + given(fileProps.isDirectory()).willReturn(true); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(cacheDir)) + .isInstanceOf(PreBidException.class) + .hasMessage("No write permissions for directory: " + cacheDir); + } + + @Test + public void createCacheFromDiskShouldNotFailIfCacheDirIsWriable() { + // given + final String cacheDir = tempDir.toString(); + tempDir.toFile().setWritable(true, false); + final FileProps fileProps = mock(FileProps.class); + given(fileSystem.existsBlocking(eq(cacheDir))).willReturn(true); + given(fileSystem.propsBlocking(eq(cacheDir))).willReturn(fileProps); + given(fileProps.isDirectory()).willReturn(true); + + // when and then + assertDoesNotThrow(() -> target.createCacheFromDisk(cacheDir)); + } + + @Test + public void createCacheFromDiskShouldReturnCacheLoadedFromJsonFiles() throws JsonProcessingException { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("/cache/dir/1.json")); + given(fileSystem.readFileBlocking(eq("/cache/dir/1.json"))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(givenVendorList()))); + + // when + final Map> cache = target.createCacheFromDisk(CACHE_DIR); + + // then + assertThat(cache).isEqualTo(singletonMap(1, givenVendorMap())); + } + + @Test + public void createCacheFromDiskShouldReturnEmptyCacheWhenCacheDirHasNoJsonFiles() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + final Map> cache = target.createCacheFromDisk(CACHE_DIR); + + // then + assertThat(cache).isEmpty(); + } + + @Test + public void createCacheFromDiskShouldFailIfCannotReadCacheDir() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void createCacheFromDiskShouldFailIfCannotReadAtLeastOneVendorListFile() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("1.json")); + given(fileSystem.readFileBlocking(eq("1.json"))).willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void createCacheFromDiskShouldFailIfAtLeastOneVendorListFileCannotBeParsed() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("1.json")); + given(fileSystem.readFileBlocking(eq("1.json"))).willReturn(Buffer.buffer("invalid")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + + @Test + public void saveToFileShouldCompleteWithVendorListResultWhenWriteSucceeds() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); + final VendorListResult vendorListResult = VendorListResult.of(1, vendorListAsString, vendorList); + given(fileSystem.writeFile(anyString(), any(Buffer.class))).willReturn(Future.succeededFuture()); + + // when + final Future future = target.saveToFile(vendorListResult, CACHE_DIR, GENERATION_VERSION); + + // then + org.prebid.server.assertion.FutureAssertion.assertThat(future).succeededWith(vendorListResult); + } + + @Test + public void saveToFileShouldWriteToExpectedPathWithExpectedContent() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); + final VendorListResult vendorListResult = VendorListResult.of(1, vendorListAsString, vendorList); + given(fileSystem.writeFile(anyString(), any(Buffer.class))).willReturn(Future.succeededFuture()); + + // when + target.saveToFile(vendorListResult, CACHE_DIR, GENERATION_VERSION); + + // then + final ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(String.class); + final ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(Buffer.class); + verify(fileSystem).writeFile(pathCaptor.capture(), bufferCaptor.capture()); + assertThat(pathCaptor.getValue()).isEqualTo(new File(CACHE_DIR, "1.json").getPath()); + assertThat(bufferCaptor.getValue().toString()).isEqualTo(vendorListAsString); + } + + @Test + public void saveToFileShouldFailWhenWriteFails() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); + final VendorListResult vendorListResult = VendorListResult.of(1, vendorListAsString, vendorList); + final RuntimeException exception = new RuntimeException("write error"); + given(fileSystem.writeFile(anyString(), any(Buffer.class))).willReturn(Future.failedFuture(exception)); + + // when + final Future future = target.saveToFile(vendorListResult, CACHE_DIR, GENERATION_VERSION); + + // then + org.prebid.server.assertion.FutureAssertion.assertThat(future) + .isFailed() + .isInstanceOf(RuntimeException.class) + .hasMessage("write error"); + } + + @Test + public void readFallbackVendorListShouldReturnNullWhenPathIsBlank() { + // when and then + assertThat(target.readFallbackVendorList(null)).isNull(); + assertThat(target.readFallbackVendorList("")).isNull(); + assertThat(target.readFallbackVendorList(" ")).isNull(); + } + + @Test + public void readFallbackVendorListShouldReturnVendorsWhenPathIsValid() throws JsonProcessingException { + // given + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(givenVendorList()))); + + // when + final Map vendors = target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH); + + // then + assertThat(vendors).isEqualTo(givenVendorMap()); + } + + @Test + public void readFallbackVendorListShouldFailIfCannotReadFallbackFile() { + // given + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void readFallbackVendorListShouldFailIfFallbackCannotBeParsed() { + // given + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willReturn(Buffer.buffer("invalid")); + + // when and then + assertThatThrownBy(() -> target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + + @Test + public void getLatestVendorListFromCacheShouldReturnEmptyWhenCacheDirHasNoJsonFiles() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + final Optional result = target.getLatestVendorListFromCache(CACHE_DIR); + + // then + assertThat(result).isEmpty(); + } + + @Test + public void getLatestVendorListFromCacheShouldReturnVendorListWithHighestVersion() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(10); + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of( + "/cache/dir/2.json", + "/cache/dir/10.json")); + given(fileSystem.readFileBlocking(eq("/cache/dir/10.json"))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(vendorList))); + + // when + final Optional result = target.getLatestVendorListFromCache(CACHE_DIR); + + // then + assertThat(result).hasValue(vendorList); + verify(fileSystem, never()).readFileBlocking(eq("/cache/dir/2.json")); + } + + @Test + public void getLatestVendorListFromCacheShouldCreateCacheDirWhenItDoesNotExist() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + target.getLatestVendorListFromCache(CACHE_DIR); + + // then + verify(fileSystem).mkdirsBlocking(eq(CACHE_DIR)); + } + + @Test + public void getLatestVendorListFromCacheShouldFailIfCannotReadCacheDir() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.getLatestVendorListFromCache(CACHE_DIR)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void getLatestVendorListFromCacheShouldFailIfLatestVendorListFileCannotBeParsed() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("/cache/dir/1.json")); + given(fileSystem.readFileBlocking(eq("/cache/dir/1.json"))).willReturn(Buffer.buffer("invalid")); + + // when and then + assertThatThrownBy(() -> target.getLatestVendorListFromCache(CACHE_DIR)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + + @Test + public void readFallbackVendorListShouldFailIfFallbackHasInvalidData() throws JsonProcessingException { + // given + final VendorList invalidVendorList = VendorList.of(1, new Date(), emptyMap()); + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(invalidVendorList))); + + // when and then + assertThatThrownBy(() -> target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH)) + .isInstanceOf(PreBidException.class) + .hasMessageStartingWith("Fallback vendor list parsed but has invalid data:"); + } + + private static VendorList givenVendorList() { + return givenVendorList(1); + } + + private static VendorList givenVendorList(int version) { + return VendorList.of(version, new Date(), givenVendorMap()); + } + + private static Map givenVendorMap() { + final Vendor vendor = Vendor.builder() + .id(52) + .purposes(EnumSet.of(ONE)) + .legIntPurposes(EnumSet.of(TWO)) + .flexiblePurposes(EnumSet.noneOf(PurposeCode.class)) + .specialPurposes(EnumSet.noneOf(SpecialPurpose.class)) + .features(EnumSet.noneOf(Feature.class)) + .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) + .build(); + return singletonMap(52, vendor); + } +} diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java index 3687b1a6129..854db8fcef0 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java @@ -3,8 +3,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import io.vertx.core.Future; import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.file.FileSystem; import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -24,7 +22,6 @@ import org.prebid.server.vertx.httpclient.HttpClient; import org.prebid.server.vertx.httpclient.model.HttpClientResponse; -import java.io.File; import java.util.Date; import java.util.EnumSet; import java.util.HashMap; @@ -32,7 +29,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; -import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -40,9 +36,11 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.BDDMockito.given; import static org.mockito.Mock.Strictness.LENIENT; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.prebid.server.assertion.FutureAssertion.assertThat; @@ -59,8 +57,6 @@ public class VendorListServiceTest extends VertxTest { @Mock private Vertx vertx; - @Mock - private FileSystem fileSystem; @Mock(strictness = LENIENT) private HttpClient httpClient; @Mock @@ -69,19 +65,17 @@ public class VendorListServiceTest extends VertxTest { private BidderCatalog bidderCatalog; @Mock(strictness = LENIENT) private VendorListFetchThrottler fetchThrottler; + @Mock(strictness = LENIENT) + private VendorListFileStore vendorListFileStore; private VendorListService target; @BeforeEach public void setUp() throws JsonProcessingException { - given(fileSystem.existsBlocking(anyString())).willReturn(false); // always create cache dir - given(bidderCatalog.knownVendorIds()).willReturn(singleton(52)); - - given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) - .willReturn(Buffer.buffer(mapper.writeValueAsString(givenVendorList()))); - given(fetchThrottler.registerFetchAttempt(anyInt())).willReturn(true); + given(vendorListFileStore.readFallbackVendorList(isNull())).willReturn(null); + given(vendorListFileStore.readFallbackVendorList(eq(FALLBACK_VENDOR_LIST_PATH))).willReturn(givenVendorMap()); target = new VendorListService( 0, @@ -92,44 +86,72 @@ public void setUp() throws JsonProcessingException { false, FALLBACK_VENDOR_LIST_PATH, vertx, - fileSystem, httpClient, metrics, GENERATION_VERSION, - jacksonMapper, - fetchThrottler); + fetchThrottler, + vendorListFileStore, + jacksonMapper); } // Creation related tests @Test - public void creationShouldFailsIfCannotCreateCacheDir() { - // given - given(fileSystem.mkdirsBlocking(anyString())).willThrow(new RuntimeException("dir creation error")); + public void creationShouldCreateCacheFromDisk() { + reset(vendorListFileStore); + + // given and when + new VendorListService( + 0, + CACHE_DIR, + "http://vendorlist/{VERSION}", + 0, + REFRESH_MISSING_LIST_PERIOD_MS, + false, + FALLBACK_VENDOR_LIST_PATH, + vertx, + httpClient, + metrics, + GENERATION_VERSION, + fetchThrottler, + vendorListFileStore, + jacksonMapper); // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/{VERSION}", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .hasMessage("dir creation error"); + verify(vendorListFileStore).createCacheFromDisk(eq(CACHE_DIR)); + } + + @Test + public void creationShouldFailIfCannotCreateCache() { + // given + given(vendorListFileStore.createCacheFromDisk(anyString())) + .willThrow(new RuntimeException("error creating cache")); + + // when and then + assertThatThrownBy(() -> new VendorListService( + 0, + CACHE_DIR, + "http://vendorlist/{VERSION}", + 0, + REFRESH_MISSING_LIST_PERIOD_MS, + true, + null, + vertx, + httpClient, + metrics, + GENERATION_VERSION, + fetchThrottler, + vendorListFileStore, + jacksonMapper)) + .isInstanceOf(RuntimeException.class) + .hasMessage("error creating cache"); } @Test public void shouldStartUsingFallbackVersionIfDeprecatedIsTrue() { // given + given(vendorListFileStore.readFallbackVendorList(anyString())).willReturn(givenVendorMap()); + target = new VendorListService( 0, CACHE_DIR, @@ -139,12 +161,12 @@ public void shouldStartUsingFallbackVersionIfDeprecatedIsTrue() { true, FALLBACK_VENDOR_LIST_PATH, vertx, - fileSystem, httpClient, metrics, GENERATION_VERSION, - jacksonMapper, - fetchThrottler); + fetchThrottler, + vendorListFileStore, + jacksonMapper); // when final Future> future = target.forVersion(1); @@ -175,96 +197,16 @@ public void shouldThrowExceptionIfVersionIsDeprecatedAndNoFallbackPresent() { true, null, vertx, - fileSystem, httpClient, metrics, GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) + fetchThrottler, + vendorListFileStore, + jacksonMapper)) .isInstanceOf(PreBidException.class) .hasMessage("No fallback vendorList for deprecated version present"); } - @Test - public void creationShouldFailsIfCannotReadFiles() { - // given - given(fileSystem.readDirBlocking(anyString())).willThrow(new RuntimeException("read error")); - - // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/{VERSION}", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .isInstanceOf(RuntimeException.class) - .hasMessage("read error"); - } - - @Test - public void creationShouldFailsIfCannotReadAtLeastOneVendorListFile() { - // given - given(fileSystem.readDirBlocking(anyString())).willReturn(singletonList("1.json")); - given(fileSystem.readFileBlocking(anyString())).willThrow(new RuntimeException("read error")); - - // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/{VERSION}", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .isInstanceOf(RuntimeException.class) - .hasMessage("read error"); - } - - @Test - public void creationShouldFailsIfAtLeastOneVendorListFileCannotBeParsed() { - // given - given(fileSystem.readDirBlocking(anyString())).willReturn(singletonList("1.json")); - given(fileSystem.readFileBlocking(anyString())).willReturn(Buffer.buffer("invalid")); - - // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/{VERSION}", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .isInstanceOf(PreBidException.class) - .hasMessage("Cannot parse vendor list from: invalid"); - } - // Http related tests @Test @@ -291,11 +233,11 @@ public void shouldNotPerformHttpRequestIfVendorListNotFoundAndFetchNotAllowed() // then verify(httpClient, never()).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfReadingHttpResponseFails() { + public void shouldNotTryToSaveFileIfReadingHttpResponseFails() { // given givenHttpClientProducesException(new RuntimeException("Response exception")); @@ -304,11 +246,11 @@ public void shouldNotAskToSaveFileIfReadingHttpResponseFails() { // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfResponseCodeIsNot200() { + public void shouldNotTryToSaveFileIfResponseCodeIsNot200() { // given givenHttpClientReturnsResponse(503, null); @@ -317,11 +259,11 @@ public void shouldNotAskToSaveFileIfResponseCodeIsNot200() { // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfResponseBodyCouldNotBeParsed() { + public void shouldNotTryToSaveFileIfResponseBodyCouldNotBeParsed() { // given givenHttpClientReturnsResponse(200, "response"); @@ -330,11 +272,11 @@ public void shouldNotAskToSaveFileIfResponseBodyCouldNotBeParsed() { // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidVendorListVersion() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasInvalidVendorListVersion() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(null, null, null); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -344,11 +286,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidVendorListVersion // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidLastUpdated() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasInvalidLastUpdated() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, null, null); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -358,11 +300,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidLastUpdated() thr // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasNoVendors() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasNoVendors() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, new Date(), null); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -372,11 +314,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasNoVendors() throws JsonP // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasEmptyVendors() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasEmptyVendors() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, new Date(), emptyMap()); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -386,11 +328,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasEmptyVendors() throws Js // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, new Date(), singletonMap(1, Vendor.builder().build())); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -400,7 +342,7 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor( // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } // File system related tests @@ -408,16 +350,20 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor( @Test public void shouldSaveFileWithExpectedPathAndContentIfVendorListNotFound() throws JsonProcessingException { // given - final String vendorListAsString = mapper.writeValueAsString(givenVendorList()); + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); givenHttpClientReturnsResponse(200, vendorListAsString); - // generate file path to avoid conflicts with path separators in different OS - final String filePath = new File("/cache/dir/1.json").getPath(); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); // then - verify(fileSystem).writeFile(eq(filePath), eq(Buffer.buffer(vendorListAsString))); + verify(vendorListFileStore).saveToFile( + eq(VendorListResult.of(1, vendorListAsString, vendorList)), + eq(CACHE_DIR), + eq(GENERATION_VERSION)); } // In-memory cache related tests @@ -454,7 +400,8 @@ public void shouldReturnVendorListFromCache() throws JsonProcessingException { // given givenHttpClientReturnsResponse(200, mapper.writeValueAsString(givenVendorList())); - given(fileSystem.writeFile(anyString(), any())).willReturn(Future.succeededFuture()); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); // populate cache @@ -501,8 +448,8 @@ public void shouldKeepPurposesForAllVendors() throws JsonProcessingException { final VendorList vendorList = VendorList.of(1, new Date(), idToVendor); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); - given(fileSystem.writeFile(anyString(), any())) - .willReturn(Future.succeededFuture()); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); // populate cache @@ -569,7 +516,8 @@ public void shouldIncrementVendorListErrorMetricWhenFileIsNotSaved() throws Json // given givenHttpClientReturnsResponse(200, mapper.writeValueAsString(givenVendorList())); - given(fileSystem.writeFile(anyString(), any())).willReturn(Future.failedFuture("error")); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willReturn(Future.failedFuture("error")); // when target.forVersion(1); @@ -583,7 +531,8 @@ public void shouldIncrementVendorListOkMetric() throws JsonProcessingException { // given givenHttpClientReturnsResponse(200, mapper.writeValueAsString(givenVendorList())); - given(fileSystem.writeFile(anyString(), any())).willReturn(Future.succeededFuture()); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); @@ -609,6 +558,10 @@ public void shouldIncrementVendorListFallbackMetric() { } private static VendorList givenVendorList() { + return VendorList.of(1, new Date(), givenVendorMap()); + } + + private static Map givenVendorMap() { final Vendor vendor = Vendor.builder() .id(52) .purposes(EnumSet.of(ONE)) @@ -618,7 +571,7 @@ private static VendorList givenVendorList() { .features(EnumSet.noneOf(Feature.class)) .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) .build(); - return VendorList.of(1, new Date(), singletonMap(52, vendor)); + return singletonMap(52, vendor); } private void givenHttpClientReturnsResponse(int statusCode, String response) { diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtilTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtilTest.java new file mode 100644 index 00000000000..8ff016e924b --- /dev/null +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtilTest.java @@ -0,0 +1,192 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.junit.jupiter.api.Test; +import org.prebid.server.VertxTest; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Feature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialFeature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialPurpose; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.util.Date; +import java.util.EnumSet; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.ONE; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.TWO; + +public class VendorListUtilTest extends VertxTest { + + @Test + public void parseVendorListShouldReturnVendorListWhenContentIsValid() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String content = mapper.writeValueAsString(vendorList); + + // when + final VendorList result = VendorListUtil.parseVendorList(content, jacksonMapper); + + // then + assertThat(result).isEqualTo(vendorList); + } + + @Test + public void parseVendorListShouldThrowExceptionWhenContentCannotBeParsed() { + // when and then + assertThatThrownBy(() -> VendorListUtil.parseVendorList("invalid", jacksonMapper)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + + @Test + public void vendorListIsValidShouldReturnTrueWhenVendorListIsValid() { + // when and then + assertThat(VendorListUtil.vendorListIsValid(givenVendorList())).isTrue(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorListVersionIsNull() { + // given + final VendorList vendorList = VendorList.of(null, new Date(), givenVendorMap()); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenLastUpdatedIsNull() { + // given + final VendorList vendorList = VendorList.of(1, null, givenVendorMap()); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorsIsNull() { + // given + final VendorList vendorList = VendorList.of(1, new Date(), null); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorsIsEmpty() { + // given + final VendorList vendorList = VendorList.of(1, new Date(), emptyMap()); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorIsNull() { + // given + final VendorList vendorList = VendorList.of(1, new Date(), singletonMap(1, null)); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorIdIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().id(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorPurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().purposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorLegIntPurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().legIntPurposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorFlexiblePurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().flexiblePurposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorSpecialPurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().specialPurposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorFeaturesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().features(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorSpecialFeaturesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().specialFeatures(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + private static VendorList givenVendorList() { + return VendorList.of(1, new Date(), givenVendorMap()); + } + + private static VendorList givenVendorListWithVendor(Vendor vendor) { + return VendorList.of(1, new Date(), singletonMap(vendor.getId() != null ? vendor.getId() : 1, vendor)); + } + + private static Map givenVendorMap() { + return singletonMap(52, givenVendor()); + } + + private static Vendor givenVendor() { + return Vendor.builder() + .id(52) + .purposes(EnumSet.of(ONE)) + .legIntPurposes(EnumSet.of(TWO)) + .flexiblePurposes(EnumSet.noneOf(PurposeCode.class)) + .specialPurposes(EnumSet.noneOf(SpecialPurpose.class)) + .features(EnumSet.noneOf(Feature.class)) + .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) + .build(); + } +} diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java index 8437698b9f0..4ca880f61e0 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java @@ -2,29 +2,47 @@ import com.iabtcf.decoder.TCString; import com.iabtcf.encoder.TCStringEncoder; +import io.vertx.core.Future; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Map; import java.util.concurrent.ThreadLocalRandom; +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static java.util.Collections.singleton; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.verify; +import static org.prebid.server.assertion.FutureAssertion.assertThat; @ExtendWith(MockitoExtension.class) public class VersionedVendorListServiceTest { - private VersionedVendorListService versionedVendorListService; + private static final Instant NOW = Instant.parse("2024-06-01T12:00:00Z"); + + private VersionedVendorListService target; @Mock private VendorListService vendorListServiceV2; @Mock private VendorListService vendorListServiceV3; + @Mock + private LiveVendorListService liveVendorListService; @BeforeEach public void setUp() { - versionedVendorListService = new VersionedVendorListService(vendorListServiceV2, vendorListServiceV3); + target = new VersionedVendorListService( + vendorListServiceV2, vendorListServiceV3, liveVendorListService, Clock.fixed(NOW, ZoneOffset.UTC)); } @Test @@ -36,9 +54,10 @@ public void versionedVendorListServiceShouldTreatTcfPolicyLessThanFourAsVendorLi .tcfPolicyVersion(tcfPolicyVersion) .vendorListVersion(12) .toTCString(); + given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(emptyMap())); // when - versionedVendorListService.forConsent(consent); + target.forConsent(consent); // then verify(vendorListServiceV2).forVersion(12); @@ -53,11 +72,64 @@ public void versionedVendorListServiceShouldTreatTcfPolicyGreaterOrEqualFourAsVe .tcfPolicyVersion(tcfPolicyVersion) .vendorListVersion(12) .toTCString(); + given(vendorListServiceV3.forVersion(anyInt())).willReturn(Future.succeededFuture(emptyMap())); // when - versionedVendorListService.forConsent(consent); + target.forConsent(consent); // then verify(vendorListServiceV3).forVersion(12); } + + @Test + public void forConsentShouldRemoveVendorsMarkedDeletedInRequestedGvl() { + // given + final Vendor deletedVendor = Vendor.empty(2).toBuilder() + .deletedDate(Instant.parse("2024-01-01T00:00:00Z")) + .build(); + final Vendor activeVendor = Vendor.empty(52); + final Map vendorList = Map.of(2, deletedVendor, 52, activeVendor); + final TCString consent = TCStringEncoder.newBuilder() + .version(2) + .tcfPolicyVersion(3) + .vendorListVersion(12) + .toTCString(); + + given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(vendorList)); + given(liveVendorListService.getDeletedVendorIds()).willReturn(emptySet()); + + // when and then + assertThat(target.forConsent(consent)) + .isSucceeded() + .unwrap() + .satisfies(result -> { + assertThat(result.get(2)).isNull(); + assertThat(result.get(52)).isSameAs(activeVendor); + }); + } + + @Test + public void forConsentShouldRemoveVendorsMarkedDeletedInLiveGvl() { + // given + final Vendor deletedVendor = Vendor.empty(1); + final Vendor activeVendor = Vendor.empty(52); + final Map vendorList = Map.of(1, deletedVendor, 52, activeVendor); + final TCString consent = TCStringEncoder.newBuilder() + .version(2) + .tcfPolicyVersion(3) + .vendorListVersion(12) + .toTCString(); + + given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(vendorList)); + given(liveVendorListService.getDeletedVendorIds()).willReturn(singleton(1)); + + // when and then + assertThat(target.forConsent(consent)) + .isSucceeded() + .unwrap() + .satisfies(result -> { + assertThat(result.get(1)).isNull(); + assertThat(result.get(52)).isSameAs(activeVendor); + }); + } } diff --git a/src/test/resources/org/prebid/server/it/test-application.properties b/src/test/resources/org/prebid/server/it/test-application.properties index 9df120154d8..8c73beb8da8 100644 --- a/src/test/resources/org/prebid/server/it/test-application.properties +++ b/src/test/resources/org/prebid/server/it/test-application.properties @@ -759,4 +759,5 @@ gdpr.host-vendor-id=1 gdpr.default-value=1 gdpr.vendorlist.v2.cache-dir=src/test/resources/org/prebid/server/it/gdpr-vendorlist2 gdpr.vendorlist.v3.cache-dir=src/test/resources/org/prebid/server/it/gdpr-vendorlist3 +gdpr.vendorlist.live.startup-cache-dir=src/test/resources/org/prebid/server/it/gdpr-vendorlist3 ccpa.enforce=false